From d8d00cc73825d86aa5fe411ab73bab46f57158c1 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 26 Apr 2026 16:59:05 -0700 Subject: [PATCH 1/3] Remove MVTC experimental flag --- .../labkey/api/exp/property/DomainUtil.java | 2 - api/src/org/labkey/api/settings/AppProps.java | 519 ++-- .../labkey/experiment/ExperimentModule.java | 2375 ++++++++-------- .../labkey/test/tests/study/AssayTest.java | 2443 ++++++++--------- .../test/tests/study/StudyDatasetsTest.java | 1 - 5 files changed, 2666 insertions(+), 2674 deletions(-) diff --git a/api/src/org/labkey/api/exp/property/DomainUtil.java b/api/src/org/labkey/api/exp/property/DomainUtil.java index 933a0fb33aa..dab9d237276 100644 --- a/api/src/org/labkey/api/exp/property/DomainUtil.java +++ b/api/src/org/labkey/api/exp/property/DomainUtil.java @@ -444,8 +444,6 @@ public static boolean allowMultiChoice(DomainKind kind) { if (!kind.allowMultiChoiceProperties()) return false; - if (!OptionalFeatureService.get().isFeatureEnabled(AppProps.MULTI_VALUE_TEXT_CHOICE)) - return false; return CoreSchema.getInstance().getSqlDialect().isPostgreSQL(); } diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index 3ac3f6c8c37..7ebe2bd9266 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -1,260 +1,259 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.settings; - -import jakarta.servlet.http.HttpServletRequest; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.SupportedDatabase; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.Path; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.view.ActionURL; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Stores basic site-wide configuration. - * @see org.labkey.api.settings.WriteableAppProps - */ -public interface AppProps -{ - AppProps _instance = new AppPropsImpl(); - - String SCOPE_SITE_SETTINGS = "SiteSettings"; - - // Used for all optional features; "experimental" for historical reasons. - String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; - String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. - String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; - String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; - String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; - String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; - String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; - String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; - String MULTI_VALUE_TEXT_CHOICE = "multiChoiceDataType"; - - String UNKNOWN_VERSION = "Unknown Release Version"; - - static AppProps getInstance() - { - return _instance; - } - - static WriteableAppProps getWriteableInstance() - { - return new WriteableAppProps(ContainerManager.getRoot()); - } - - String getServerSessionGUID(); - - boolean isMailRecorderEnabled(); - - boolean isOptionalFeatureEnabled(String feature); - - boolean isDevMode(); - - @Nullable - String getEnlistmentId(); - - boolean isCachingAllowed(); - - boolean isRecompileJspEnabled(); - - /** - * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode - * without the risk of loading unwanted resources from a source tree that may not match the deployed server. - * - * WARNING: Setting this flag will interfere with the population of module beans, resulting in a - * mismatch between deployed modules and their properties on the server. - * - * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false - * - * @see org.labkey.api.module.DefaultModule#setSourcePath(String) - * @see org.labkey.api.module.DefaultModule#setBuildPath(String) - * @see DefaultModule#computeResourceDirectory() - */ - boolean isIgnoreModuleSource(); - - void setProjectRoot(String projectRoot); - - /** - * @return the root of the main source tree - */ - @Nullable - String getProjectRoot(); - - /** - * @return directory under which all containers will automatically have their own subdirectory for storing files - */ - @Nullable - File getFileSystemRoot(); - - @NotNull - UsageReportingLevel getUsageReportingLevel(); - - /** - * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". - * Or "Unknown Release Version". - */ - @NotNull - String getReleaseVersion(); - - /** - * Convenience method for getting the core schema version, returning 0.0 instead of null - */ - double getSchemaVersion(); - - String getContextPath(); - - Path getParsedContextPath(); - - int getServerPort(); - - String getScheme(); - - String getServerName(); - - /** - * Save the current request URL if the base server URL property is not set - */ - void ensureBaseServerUrl(HttpServletRequest request); - - void setContextPath(String contextPath); - - boolean isSetBaseServerUrl(); - - String getBaseServerUrl(); - - String getHomePageUrl(); - - ActionURL getHomePageActionURL(); - - String getSiteWelcomePageUrlString(); - - int getLookAndFeelRevision(); - - String getDefaultLsidAuthority(); - - String getPipelineToolsDirectory(); - - boolean isSSLRequired(); - - boolean isUserRequestedAdminOnlyMode(); - - String getAdminOnlyMessage(); - - boolean isShowRibbonMessage(); - - @Nullable String getRibbonMessage(); - - int getSSLPort(); - - int getMemoryUsageDumpInterval(); - - /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ - int getReadOnlyHttpRequestTimeout(); - - int getMaxBLOBSize(); - - boolean isExt3Required(); - - boolean isExt3APIRequired(); - - ExceptionReportingLevel getExceptionReportingLevel(); - - /** - * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full - * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. - * - * @return if navigation access is open - */ - boolean isNavigationAccessOpen(); - - boolean isSelfReportExceptions(); - - String getServerGUID(); - - String getBLASTServerBaseURL(); - - /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ - String getWebappConfigurationFilename(); - - /** - * Email address of the primary site or application administrator, set on the site settings page. Useful in error - * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., - * only impersonating troubleshooters). - * - * @return Email address of the primary site or application administrator - */ - @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); - - boolean isAllowApiKeys(); - - int getApiKeyExpirationSeconds(); - - boolean isAllowSessionKeys(); - - // configurable http security settings - - /** - * @return "SAMEORIGIN" or "DENY" or "ALLOW" - */ - String getXFrameOption(); - - String getStaticFilesPrefix(); - - boolean isWebfilesRootEnabled(); - - boolean isFileUploadDisabled(); - - boolean isInvalidFilenameUploadBlocked(); - - boolean isInvalidFilenameBlocked(); - - /** @return whether the server should include its name and version as a header in HTTP responses */ - boolean isIncludeServerHttpHeader(); - - /** - * @return List of configured external redirect hosts - */ - @NotNull - List getExternalRedirectHosts(); - - /** - * @return List of configured external resource hosts - */ - @Deprecated // Left for upgrade code only - @NotNull - List getExternalSourceHosts(); - - Map getStashedStartupProperties(); - - @NotNull String getDistributionName(); - - @NotNull String getDistributionFilename(); - - @NotNull Set getDistributionSupportedDatabases(); - - @NotNull List getAllowedExtensions(); - - @NotNull String getAllowedExternalResourceHosts(); -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.settings; + +import jakarta.servlet.http.HttpServletRequest; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.SupportedDatabase; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.Path; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.view.ActionURL; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Stores basic site-wide configuration. + * @see org.labkey.api.settings.WriteableAppProps + */ +public interface AppProps +{ + AppProps _instance = new AppPropsImpl(); + + String SCOPE_SITE_SETTINGS = "SiteSettings"; + + // Used for all optional features; "experimental" for historical reasons. + String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; + String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. + String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; + String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; + String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; + String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; + String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; + + String UNKNOWN_VERSION = "Unknown Release Version"; + + static AppProps getInstance() + { + return _instance; + } + + static WriteableAppProps getWriteableInstance() + { + return new WriteableAppProps(ContainerManager.getRoot()); + } + + String getServerSessionGUID(); + + boolean isMailRecorderEnabled(); + + boolean isOptionalFeatureEnabled(String feature); + + boolean isDevMode(); + + @Nullable + String getEnlistmentId(); + + boolean isCachingAllowed(); + + boolean isRecompileJspEnabled(); + + /** + * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode + * without the risk of loading unwanted resources from a source tree that may not match the deployed server. + * + * WARNING: Setting this flag will interfere with the population of module beans, resulting in a + * mismatch between deployed modules and their properties on the server. + * + * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false + * + * @see org.labkey.api.module.DefaultModule#setSourcePath(String) + * @see org.labkey.api.module.DefaultModule#setBuildPath(String) + * @see DefaultModule#computeResourceDirectory() + */ + boolean isIgnoreModuleSource(); + + void setProjectRoot(String projectRoot); + + /** + * @return the root of the main source tree + */ + @Nullable + String getProjectRoot(); + + /** + * @return directory under which all containers will automatically have their own subdirectory for storing files + */ + @Nullable + File getFileSystemRoot(); + + @NotNull + UsageReportingLevel getUsageReportingLevel(); + + /** + * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". + * Or "Unknown Release Version". + */ + @NotNull + String getReleaseVersion(); + + /** + * Convenience method for getting the core schema version, returning 0.0 instead of null + */ + double getSchemaVersion(); + + String getContextPath(); + + Path getParsedContextPath(); + + int getServerPort(); + + String getScheme(); + + String getServerName(); + + /** + * Save the current request URL if the base server URL property is not set + */ + void ensureBaseServerUrl(HttpServletRequest request); + + void setContextPath(String contextPath); + + boolean isSetBaseServerUrl(); + + String getBaseServerUrl(); + + String getHomePageUrl(); + + ActionURL getHomePageActionURL(); + + String getSiteWelcomePageUrlString(); + + int getLookAndFeelRevision(); + + String getDefaultLsidAuthority(); + + String getPipelineToolsDirectory(); + + boolean isSSLRequired(); + + boolean isUserRequestedAdminOnlyMode(); + + String getAdminOnlyMessage(); + + boolean isShowRibbonMessage(); + + @Nullable String getRibbonMessage(); + + int getSSLPort(); + + int getMemoryUsageDumpInterval(); + + /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ + int getReadOnlyHttpRequestTimeout(); + + int getMaxBLOBSize(); + + boolean isExt3Required(); + + boolean isExt3APIRequired(); + + ExceptionReportingLevel getExceptionReportingLevel(); + + /** + * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full + * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. + * + * @return if navigation access is open + */ + boolean isNavigationAccessOpen(); + + boolean isSelfReportExceptions(); + + String getServerGUID(); + + String getBLASTServerBaseURL(); + + /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ + String getWebappConfigurationFilename(); + + /** + * Email address of the primary site or application administrator, set on the site settings page. Useful in error + * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., + * only impersonating troubleshooters). + * + * @return Email address of the primary site or application administrator + */ + @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); + + boolean isAllowApiKeys(); + + int getApiKeyExpirationSeconds(); + + boolean isAllowSessionKeys(); + + // configurable http security settings + + /** + * @return "SAMEORIGIN" or "DENY" or "ALLOW" + */ + String getXFrameOption(); + + String getStaticFilesPrefix(); + + boolean isWebfilesRootEnabled(); + + boolean isFileUploadDisabled(); + + boolean isInvalidFilenameUploadBlocked(); + + boolean isInvalidFilenameBlocked(); + + /** @return whether the server should include its name and version as a header in HTTP responses */ + boolean isIncludeServerHttpHeader(); + + /** + * @return List of configured external redirect hosts + */ + @NotNull + List getExternalRedirectHosts(); + + /** + * @return List of configured external resource hosts + */ + @Deprecated // Left for upgrade code only + @NotNull + List getExternalSourceHosts(); + + Map getStashedStartupProperties(); + + @NotNull String getDistributionName(); + + @NotNull String getDistributionFilename(); + + @NotNull Set getDistributionSupportedDatabases(); + + @NotNull List getAllowedExtensions(); + + @NotNull String getAllowedExternalResourceHosts(); +} diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index efcb9f2c1fc..966f924dbd2 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1189 +1,1186 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AbstractAssayProvider; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.assay.transform.DataTransformService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.api.writer.ContainerUser; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.SampleTypeUpdateServiceDI; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; -import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 26.005; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false, true); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", - "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); - - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.MULTI_VALUE_TEXT_CHOICE, "Allow multi-value Text Choice properties", - "Support selecting more than one value for text choice fields", false); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", - "Support for querying lineage of experiment objects", false, true); - OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", - "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false, true); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString() - ).getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString(), - "%\"" + DataTransformService.TransformOperation.UPDATE + "\"%" - ).getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString(), - "%\"" + DataTransformService.TransformOperation.INSERT + "\"%" - ).getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - // metric to count the number of Luminex and Standard assay runs that were imported with > 1 data file - assayMetrics.put("assayRunsWithMultipleInputFiles", new SqlSelector(schema, """ - SELECT COUNT(*) FROM ( - SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data - WHERE lsid NOT LIKE '%:RelatedFile.%' AND sourceapplicationid IN ( - SELECT rowid FROM exp.protocolapplication - WHERE lsid LIKE '%:SimpleProtocol.CoreStep' AND (protocollsid LIKE '%:LuminexAssayProtocol.%' OR protocollsid LIKE '%:GeneralAssayProtocol.%') - ) - GROUP BY sourceapplicationid - ) x WHERE count > 1""").getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); - results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); - results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithMultiValueColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithMultiValueColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - } - - @Override - public void registerMigrationHandlers(@NotNull DatabaseMigrationService service) - { - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - service.registerSchemaHandler(handler); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - service.registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = super.getPageContextJson(context); - json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); - return json; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AbstractAssayProvider; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.api.writer.ContainerUser; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 26.005; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false, true); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", + "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); + + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", + "Support for querying lineage of experiment objects", false, true); + OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", + "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false, true); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString() + ).getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString(), + "%\"" + DataTransformService.TransformOperation.UPDATE + "\"%" + ).getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString(), + "%\"" + DataTransformService.TransformOperation.INSERT + "\"%" + ).getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + // metric to count the number of Luminex and Standard assay runs that were imported with > 1 data file + assayMetrics.put("assayRunsWithMultipleInputFiles", new SqlSelector(schema, """ + SELECT COUNT(*) FROM ( + SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data + WHERE lsid NOT LIKE '%:RelatedFile.%' AND sourceapplicationid IN ( + SELECT rowid FROM exp.protocolapplication + WHERE lsid LIKE '%:SimpleProtocol.CoreStep' AND (protocollsid LIKE '%:LuminexAssayProtocol.%' OR protocollsid LIKE '%:GeneralAssayProtocol.%') + ) + GROUP BY sourceapplicationid + ) x WHERE count > 1""").getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); + results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithMultiValueColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithMultiValueColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + } + + @Override + public void registerMigrationHandlers(@NotNull DatabaseMigrationService service) + { + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + service.registerSchemaHandler(handler); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + service.registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = super.getPageContextJson(context); + json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); + return json; + } +} diff --git a/study/test/src/org/labkey/test/tests/study/AssayTest.java b/study/test/src/org/labkey/test/tests/study/AssayTest.java index a1285d1436f..ab4555ddf0c 100644 --- a/study/test/src/org/labkey/test/tests/study/AssayTest.java +++ b/study/test/src/org/labkey/test/tests/study/AssayTest.java @@ -1,1222 +1,1221 @@ -/* - * Copyright (c) 2016-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.test.tests.study; - -import org.assertj.core.api.Assertions; -import org.junit.Assume; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.labkey.api.util.FileUtil; -import org.labkey.remoteapi.CommandException; -import org.labkey.remoteapi.assay.AssayListCommand; -import org.labkey.remoteapi.assay.AssayListResponse; -import org.labkey.test.Locator; -import org.labkey.test.TestFileUtils; -import org.labkey.test.TestTimeoutException; -import org.labkey.test.WebTestHelper; -import org.labkey.test.categories.Assays; -import org.labkey.test.categories.Daily; -import org.labkey.test.components.CustomizeView; -import org.labkey.test.components.assay.AssayConstants; -import org.labkey.test.components.domain.DomainFieldRow; -import org.labkey.test.components.domain.DomainFormPanel; -import org.labkey.test.pages.ReactAssayDesignerPage; -import org.labkey.test.pages.assay.AssayBeginPage; -import org.labkey.test.pages.assay.AssayImportPage; -import org.labkey.test.pages.assay.AssayRunsPage; -import org.labkey.test.params.FieldDefinition; -import org.labkey.test.params.FieldInfo; -import org.labkey.test.params.assay.GeneralAssayDesign; -import org.labkey.test.params.experiment.SampleTypeDefinition; -import org.labkey.test.tests.AbstractAssayTest; -import org.labkey.test.tests.AuditLogTest; -import org.labkey.test.util.AuditLogHelper; -import org.labkey.test.util.DataRegionTable; -import org.labkey.test.util.DomainUtils; -import org.labkey.test.util.LogMethod; -import org.labkey.test.util.OptionalFeatureHelper; -import org.labkey.test.util.SampleTypeHelper; -import org.labkey.test.util.StudyHelper; -import org.labkey.test.util.TestDataGenerator; -import org.labkey.test.util.data.TestArrayDataUtils; -import org.labkey.test.util.data.TestDataUtils; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.labkey.test.util.TestDataGenerator.randomTextChoice; -import static org.labkey.test.util.TestDataGenerator.shuffleSelect; -import static org.labkey.test.util.data.TestArrayDataUtils.formatMultiValueText; - -@Category({Daily.class, Assays.class}) -public class AssayTest extends AbstractAssayTest -{ - private static final String INVESTIGATOR = "Dr. No"; - private static final String GRANT = "SPECTRE"; - private static final String DESCRIPTION = "World Domination."; - private static final String ISSUE_53625_ASSAY = TestDataGenerator.randomDomainName("Issue53625", DomainUtils.DomainKind.Assay); - private static final String ISSUE_53625_PROJECT = "Issue53625Project"; - private static final String ISSUE_53616_ASSAY = "Issue53616Assay"; - private static final String ISSUE_53616_PROJECT = "Issue53616Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String ISSUE_53831_PROJECT = "Issue53831Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String SAMPLE_FIELD_TEST_ASSAY = "SampleFieldTestAssay"; - private static final String SAMPLE_FIELD_PROJECT_NAME = "Sample Field Test Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String MVTC_MULTI_FILE_IMPORT_ASSAY = TestDataGenerator.randomDomainName("MVTCMultiFileImportAssay", DomainUtils.DomainKind.Assay); - private static final String MVTC_MULTI_FILE_IMPORT_PROJECT = "MVTCMultiFileImportAssay" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String COL_ASSAY_ID_LABEL = "Assay ID"; - private static final List TEXT_MULTI_CHOICE_LIST = randomTextChoice(10); - private static final FieldInfo COL_MULTITEXTCHOICE = FieldInfo.random("Multi Choice", FieldDefinition.ColumnType.MultiValueTextChoice) - .customizeFieldDefinition(fd -> fd.setMultiChoiceValues(TEXT_MULTI_CHOICE_LIST)); - - - @Override - protected String getProjectName() - { - return TEST_ASSAY_PRJ_SECURITY; - } - - /** - * Cleanup entry point. - */ - @Override - protected void doCleanup(boolean afterTest) throws TestTimeoutException - { - //should also delete the groups - _containerHelper.deleteProject(getProjectName(), false); - _containerHelper.deleteProject(SAMPLE_FIELD_PROJECT_NAME, false); - _containerHelper.deleteProject(ISSUE_53616_PROJECT, false); - _containerHelper.deleteProject(ISSUE_53625_PROJECT, false); - _containerHelper.deleteProject(ISSUE_53831_PROJECT, false); - _containerHelper.deleteProject(MVTC_MULTI_FILE_IMPORT_PROJECT, false); - - _userHelper.deleteUsers(false, TEST_ASSAY_USR_PI1, TEST_ASSAY_USR_TECH1); - } - - // Issue 53831: Assay name max length check - @Test - public void testAssayNameMaxLength() throws Exception - { - _containerHelper.createProject(ISSUE_53831_PROJECT, "Assay"); - goToProjectHome(ISSUE_53831_PROJECT); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", "a" + "0123456789".repeat(15)); - List errors = assayDesignerPage.clickSaveExpectingErrors(); - checker().verifyEquals("Wrong number of errors", 1, errors.size()); - checker().verifyEquals("Wrong error message: " + errors.get(0), - "Value is too long for assay design name, a maximum length of 150 is allowed. The supplied value, 'a01234567890123456789012...78901234567890123456789', was 151 characters long.", - errors.get(0)); - assayDesignerPage.clickCancel(); - } - - @Test - public void testAssayMultiFileImportForMVTC() throws Exception - { - Assume.assumeTrue("Multi-choice text fields are only supported on PostgreSQL", WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL); - OptionalFeatureHelper.enableOptionalFeature(getCurrentTest().createDefaultConnection(), "multiChoiceDataType"); - _containerHelper.createProject(MVTC_MULTI_FILE_IMPORT_PROJECT, "Assay"); - new GeneralAssayDesign(MVTC_MULTI_FILE_IMPORT_ASSAY) - .setRunFields(List.of(new FieldDefinition("runText", FieldDefinition.ColumnType.String)), true) - .setDataFields(List.of(COL_MULTITEXTCHOICE.getFieldDefinition()), false) - .createAssay(MVTC_MULTI_FILE_IMPORT_PROJECT, createDefaultConnection()); - - String firstFileName = "MVTCAssayImport.tsv"; - String secondFileName = "MVTCAssayImportSecond.tsv"; - List> fileDataFirstImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) - .limit(5) - .toList(); - List> fileDataSecondImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) - .limit(5) - .toList(); - - log("Import first and second runs with MVTC data from files"); - AssayImportPage assayImportPage = goToManageAssays() - .clickAssay(MVTC_MULTI_FILE_IMPORT_ASSAY) - .clickImportData(); - assayImportPage.clickNext(); - assayImportPage.setDataFile(writeMultiValueFileForAssayRun(firstFileName, fileDataFirstImport)); - - assayImportPage = assayImportPage.clickSaveAndImportAnother(); - assayImportPage.setDataFile(writeMultiValueFileForAssayRun(secondFileName, fileDataSecondImport)); - assayImportPage.clickSaveAndFinish(); - - AssayRunsPage assayRunsPage = new AssayRunsPage(getDriver()); - checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.getTable().getColumnDataAsText(COL_ASSAY_ID_LABEL)) - .as("expect both runs to appear in the runs list") - .containsExactlyInAnyOrder(firstFileName, secondFileName)); - - List expectedValues = Stream.concat(fileDataFirstImport.stream(), fileDataSecondImport.stream()) - .map(values -> TestArrayDataUtils.sortAndJoin(values, " ")) - .toList(); - checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.clickViewResults().getDataTable().getColumnDataAsText(COL_MULTITEXTCHOICE)) - .as("expect MVTC values to match imported data") - .containsExactlyInAnyOrderElementsOf(expectedValues)); - } - - private File writeMultiValueFileForAssayRun(String fileName, List> fileData) throws IOException - { - List> rows = Stream.concat( - Stream.of(List.of(COL_MULTITEXTCHOICE.getName())), - fileData.stream().map(row -> List.of(formatMultiValueText(row))) - ).toList(); - return TestDataUtils.writeRowsToFile(fileName, rows); - } - - // Issue 53616: Assay creation attempt after an error results in "Assay protocol already exists for this name." - @Test - public void testFailedCreation() throws Exception - { - _containerHelper.createProject(ISSUE_53616_PROJECT, "Assay"); - goToProjectHome(ISSUE_53616_PROJECT); - - log("Create test assay"); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", ISSUE_53616_ASSAY) - .setDescription(TEST_ASSAY_DESC); - - DomainFormPanel resultsPanel = assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset result fields - resultsPanel.addField("TooLongFieldName".repeat(20)); - - log("Save initial assay design with sample field set to 'All Samples'"); - List errors = assayDesignerPage.clickSaveExpectingErrors(); - assertEquals("Wrong number of errors", 1, errors.size()); - assertTrue("Wrong error message: " + errors.get(0), errors.get(0).startsWith("Name cannot exceed 200 characters, but was")); - - resultsPanel.removeAllFields(false); - resultsPanel.addField("ShortAndSweet"); - assayDesignerPage.clickFinish(); - - AssayListCommand command = new AssayListCommand(); - AssayListResponse response = command.execute(createDefaultConnection(), ISSUE_53616_PROJECT); - assertNotNull("Didn't find expected assay design", response.getDefinition(ISSUE_53616_ASSAY)); - } - - /** - * Performs the Assay security test - * This test creates a project with a folder hierarchy with multiple groups and users; - * defines an Assay at the project level; uploads run data as a labtech; publishes - * as a PI, and tests to make sure that security is properly enforced - */ - @Test - public void testAssaySecurity() throws Exception - { - log("Starting Assay security scenario tests"); - setupEnvironment(); - setupPipeline(getProjectName()); - SpecimenImporter importer = new SpecimenImporter(TestFileUtils.getTestTempDir(), StudyHelper.SPECIMEN_ARCHIVE_A, FileUtil.appendName(TestFileUtils.getTestTempDir(), "specimensSubDir"), TEST_ASSAY_FLDR_STUDY2, 1); - importer.importAndWaitForComplete(); - defineAssay(); - uploadRuns(TEST_ASSAY_FLDR_LAB1, TEST_ASSAY_USR_TECH1); - editResults(); - publishData(); - publishDataToDateBasedStudy(); - publishDataToVisitBasedStudy(); - editAssay(); - viewCrossFolderData(); - verifyStudyList(); - verifyRunDeletionRecallsDatasetRows(); - verifyWebdavTree(); - } - - @Test - public void testSampleFieldUpdate() - { - log("Starting sample field update test"); - _containerHelper.createProject(SAMPLE_FIELD_PROJECT_NAME, "Assay"); - - log("Create test assay"); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", SAMPLE_FIELD_TEST_ASSAY) - .setDescription(TEST_ASSAY_DESC); - - assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset batch fields - - DomainFormPanel resultsPanel = assayDesignerPage.goToResultsFields().removeAllFields(false); //remove preset result fields - - String sampleFieldName = "SampleField"; - resultsPanel.manuallyDefineFields(sampleFieldName) - .setType(FieldDefinition.ColumnType.Sample) - .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); - - log("Save initial assay design with sample field set to 'All Samples'"); - assayDesignerPage.clickFinish(); - - log("Verify save successful"); - assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); - AssayBeginPage assayPage = goToManageAssays(); - assertElementPresent(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - - log("Create new Sample Types to verify against"); - String targetTypeName = "Target Sample Type"; - SampleTypeDefinition targetDefinition = new SampleTypeDefinition(targetTypeName).setFields(new ArrayList<>()); - SampleTypeHelper ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); - ssHelper.createSampleType(targetDefinition, "Name\nS_1\nS_2\nS_3"); - - String otherTypeName = "Other Sample Type"; - SampleTypeDefinition otherDefinition = new SampleTypeDefinition(otherTypeName).setFields(new ArrayList<>()); - ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); - ssHelper.createSampleType(otherDefinition, "Name\nOS_1\nOS_2"); - - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN1, "SampleField\nOS_1"); - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); - - log("Edit assay design and change Sample field to point to created Sample Type"); - goToManageAssays(); - clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.expandFieldsPanel("Results") - .getField(sampleFieldName) - .setSampleType(targetTypeName); - designerPage.clickFinish(); - - log("Verify updates saved successfully"); - assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN2, "SampleField\nS_1"); - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable table = new DataRegionTable("Data", getDriver()); - List sampleFieldValues = table.getColumnDataAsText("SampleField"); - assertTrue("First sample should not resolve to sample type", sampleFieldValues.get(0).startsWith("<")); - assertEquals("Second sample should resolve to sample type", "S_1", sampleFieldValues.get(1)); - assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); - - log("GitHub Issue #688: verify sample lookup to createdBy"); - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("SampleField/CreatedBy"); - _customizeViewsHelper.applyCustomView(); - table = new DataRegionTable("Data", getDriver()); - List createdByValues = table.getColumnDataAsText("SampleField/CreatedBy"); - assertEquals("First sample should not have a createdBy since it doesn't resolve", " ", createdByValues.get(0)); - assertEquals("Second sample should have a createdBy since it resolves to a sample type", getCurrentUserName(), createdByValues.get(1)); - - log("Edit assay design and change Sample field to point back to 'All Samples'"); - goToManageAssays(); - clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.expandFieldsPanel("Results") - .getField(sampleFieldName) - .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); - designerPage.clickFinish(); - assertEquals("Error saving updated sample field", 0, checker().errorsSinceMark()); - - log("Verify updates saved successfully"); - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN3, "SampleField\nS_2\nOS_2"); - assertEquals("Error importing data after assay sample field update", 0, checker().errorsSinceMark()); - - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); - assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); - assertElementPresent("Sample lookup failed for: S_2", new Locator.LinkLocator("S_2"), 1); - assertElementPresent("Sample lookup failed for: OS_2", new Locator.LinkLocator("OS_2"), 1); - - log("GitHub Issue #688: verify sample lookup to createdBy"); - table = new DataRegionTable("Data", getDriver()); - for (int i = 0; i < table.getDataRowCount(); i++) - assertEquals("Row " + i + " should have current user as createdBy since they all resolve to samples", getCurrentUserName(), table.getDataAsText(i, "SampleField/CreatedBy")); - } - - private void importAssayData(String assayName, String runName, String runDataStr) - { - goToManageAssays(); - clickAndWait(Locator.linkWithText(assayName)); - clickButton("Import Data", "Run Data"); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, runName); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, runDataStr); - clickButton("Save and Finish"); - - } - - @LogMethod - private void verifyRunDeletionRecallsDatasetRows() - { - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - DataRegionTable assayRuns = new DataRegionTable("Runs", this); - assayRuns.checkCheckbox(0); - assayRuns.clickHeaderButtonAndWait("Delete"); - // Make sure that it shows that the data is part of study datasets - assertTextPresent(TEST_RUN3, "2 dataset(s)", TEST_ASSAY); - assertTextNotPresent("FirstRun"); - // Do the delete - clickButton("Confirm Delete"); - - // Be sure that we have a special audit record - clickAndWait(Locator.linkWithText("view link to study history")); - assertTextPresent("3 row(s) were recalled from a study to the assay: "); - - // Verify that the deleted run data is gone from the dataset - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); - clickAndWait(Locator.linkWithText("1 dataset")); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - assertTextPresent("AAA07XMC-04", TEST_RUN1); - assertTextNotPresent("BAQ00051-09", TEST_RUN3); - } - - //Issue 12203: Incorrect files are visible from pipeline directory - private void verifyWebdavTree() - { - beginAt("_webdav"); - _fileBrowserHelper.selectFileBrowserItem(getProjectName() + "/Studies/Study 1/"); - Locator.XPathLocator l = Locator.xpath("//span[text()='@pipeline']"); - assertElementPresent(l, 1); - } - - @LogMethod - private void editResults() throws IOException, CommandException - { - // Verify that the results aren't editable by default - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - waitAndClickAndWait(Locator.linkWithText("view results")); - DataRegionTable table = new DataRegionTable("Data", getDriver()); - assertEquals("No rows should be editable", 0, DataRegionTable.updateLinkLocator().findElements(table.getComponentElement()).size()); - assertElementNotPresent(Locator.button("Delete")); - - // Edit the design to make them editable - ReactAssayDesignerPage assayDesignerPage = _assayHelper.clickEditAssayDesign(true); - assayDesignerPage.setEditableResults(true); - assayDesignerPage.clickFinish(); - - // Try an edit - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - assertEquals("Incorrect number of results shown.", 10, table.getDataRowCount()); - doAndWaitForPageToLoad(() -> dataTable.updateLink(dataTable.getRowIndex("Specimen ID", "AAA07XK5-05")).click()); - setFormElement(Locator.name("quf_SpecimenID"), "EditedSpecimenID"); - setFormElement(Locator.name("quf_VisitID"), "601.5"); - setFormElement(Locator.name("quf_testAssayDataProp5"), "notAnumber"); - clickButton("Submit"); - assertTextPresent("Could not convert value: " + "notAnumber"); - setFormElement(Locator.name("quf_testAssayDataProp5"), "514801"); - clickButton("Submit"); - assertTextPresent("EditedSpecimenID", "601.5", "514801"); - - // Try a delete - dataTable.checkCheckbox(table.getRowIndex("Specimen ID", "EditedSpecimenID")); - doAndWaitForPageToLoad(() -> - { - dataTable.clickHeaderButton("Delete"); - assertAlert("Are you sure you want to delete the selected row?"); - }); - - // Verify that the edit was audited - AuditLogHelper auditLogHelper = new AuditLogHelper(this, () -> WebTestHelper.getRemoteApiConnection(false)); - auditLogHelper.checkAuditEventDiffCount(getProjectName(), AuditLogHelper.AuditEvent.QUERY_UPDATE_AUDIT_EVENT, List.of(0/*delete*/, 3/*edit*/)); - - goToSchemaBrowser(); - viewQueryData("auditLog", "ExperimentAuditEvent"); - assertTextPresent("1 data row has been edited in " + TEST_ASSAY + "."); - - } - - /** - * Generates the text that appears in the target study drop-down for a given study name - * @param studyName name of the target study - * @return formatted string of what appears in the target study drop-down - */ - private String getTargetStudyOptionText(String studyName) - { - //the format used in the drop down is: - // /// ( Study) - return "/" + getProjectName() + "/" + TEST_ASSAY_FLDR_STUDIES + "/" + - studyName + " (" + studyName + " Study)"; - } - - /** - * Uploads run data for the centrally defined Assay while impersonating a labtech-style user - * @param folder name of the folder into which we should upload - * @param asUser the user to impersonate before uploading - */ - @LogMethod - private void uploadRuns(String folder, String asUser) - { - log("Uploading runs into folder " + folder + " as user " + asUser); - navigateToFolder(getProjectName(), folder); - impersonate(asUser); - - clickAndWait(Locator.linkWithText("Assay List")); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - - //nav trail check - assertNavTrail("Assay List", TEST_ASSAY + " Batches"); - - clickButton("Import Data"); - assertTextPresent(TEST_ASSAY_SET_PROP_NAME + "3"); - - log("Batch properties"); - clickButton("Next"); - assertTextPresent(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + " is required and must be of type Number (Double)."); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), "Bad Test"); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), "Bad Test"); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), "Bad Test"); - clickButton("Next"); - assertTextPresent( - "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.", - "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.", - "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY)."); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]); - - //ensure that the target study drop down contains Study 1 and Study 2 only and not Study 3 - //(labtech1 does not have read perms to Study 3) - waitForElement(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1))); - assertElementPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2))); - assertElementNotPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3))); - - //select Study2 as the target study (note that PI is not an Editor in this study so we can test for override case) - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); - - clickButton("Next"); - - log("Check properties set."); - assertTextPresent( - TEST_ASSAY_SET_PROPERTIES[1], - TEST_ASSAY_SET_PROPERTIES[2], - TEST_ASSAY_SET_PROPERTIES[3], - TEST_ASSAY_SET_PROPERTIES[0]); - - log("Run properties and data"); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC); - clickButton("Save and Finish"); - - assertTextPresent(TEST_ASSAY_RUN_PROP_NAME + "0 is required and must be of type Text (String)."); - assertTextPresent(PROTOCOL_DOC.getName()); - waitAndClick(Locator.linkWithText("remove")); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN1); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN1_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); - clickButton("Save and Finish"); - - Locator loc4 = Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"); - assertEquals("", getFormElement(loc4)); - assertTextPresent("Data file contained zero data rows"); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN1_DATA1); - clickButton("Save and Import Another Run"); - - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN2); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN2_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA1); - clickButton("Save and Finish"); - - assertTextPresent(PROTOCOL_DOC2.getName()); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA2); - clickButton("Save and Finish"); - - assertTextPresent("Could not convert value 'g' (String) for Double field 'VisitID'"); - assertTextPresent(PROTOCOL_DOC2.getName()); - assertEquals(TEST_RUN2, getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); - assertEquals(TEST_RUN2_COMMENTS, getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA3); - clickButton("Save and Import Another Run"); - - assertTextPresent("Missing value for required property: " + TEST_ASSAY_DATA_PROP_NAME + "6"); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA4); - clickButton("Save and Import Another Run"); - - assertEquals("", getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); - assertEquals("", getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN3); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN3_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); - clickButton("Save and Finish"); - - assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN3_DATA1); - clickButton("Save and Finish"); - - // Verify the first run did not have a file, the second run had the attached file and the third run had a file - // with a unique name. - assertTextNotPresent(PROTOCOL_DOC.getName()); - assertTextPresent(PROTOCOL_DOC2.getName()); - assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); - - log("Check out the data for one of the runs"); - assertNoLabKeyErrors(); - assertTextPresent( - TEST_ASSAY + " Runs", - TEST_ASSAY_RUN_PROP1, - TEST_ASSAY_SET_PROPERTIES[0], - TEST_ASSAY_SET_PROPERTIES[3]); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - assertElementNotPresent(Locator.tagWithText("td", "7.0")); - // Make sure that our specimen IDs resolved correctly - assertTextPresent( - "AAA07XSF-02", - "999320885", - "301", - "AAA07XK5-05", - "999320812", - "601", - TEST_ASSAY_DATA_PROP_NAME + "4", - TEST_ASSAY_DATA_PROP_NAME + "5", - TEST_ASSAY_DATA_PROP_NAME + "6", - "2000-06-06", - "0.0", - "f", - ALIASED_DATA); - - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("SpecimenID/GlobalUniqueId"); - _customizeViewsHelper.addColumn("SpecimenID/Specimen/PrimaryType"); - _customizeViewsHelper.addColumn("SpecimenID/AssayMatch"); - _customizeViewsHelper.removeColumn("Run/testAssayRunProp1"); - _customizeViewsHelper.removeColumn("Run/Batch/testAssaySetProp2"); - _customizeViewsHelper.removeColumn("testAssayDataProp4"); - _customizeViewsHelper.applyCustomView(); - - assertTextPresent("Blood (Whole)", 4); - - Locator.XPathLocator trueLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'true']"); - int totalTrues = getElementCount(trueLocator); - assertEquals(4, totalTrues); - - DataRegionTable region = new DataRegionTable("Data", this); - region.setFilter("SpecimenID", "Starts With", "AssayTestControl"); - - // verify that there are no trues showing for the assay match column that were filtered out - totalTrues = getElementCount(trueLocator); - assertEquals(0, totalTrues); - - log("Check out the data for all of the runs"); - clickAndWait(Locator.linkWithText("view results")); - region.clearAllFilters("SpecimenID"); - assertElementPresent(Locator.tagWithText("td", "7.0")); - assertElementPresent(Locator.tagWithText("td", "18")); - - assertTextPresent("Blood (Whole)", 7); - - Locator.XPathLocator falseLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'false']"); - int totalFalses = getElementCount(falseLocator); - assertEquals(3, totalFalses); - - region.setFilter("SpecimenID", "Does Not Start With", "BAQ"); - - // verify the falses have been filtered out - totalFalses = getElementCount(falseLocator); - assertEquals(0, totalFalses); - - stopImpersonating(); - } - - /** - * Impersonates the PI user and publishes the data previous uploaded. - * This will also verify that the PI cannot publish to studies for which - * the PI does not have Editor permissions. - */ - @LogMethod - private void publishData() - { - log("Prepare visit map to check PTID counts in study navigator."); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); - _studyHelper.goToManageVisits().goToImportVisitMap(); - setFormElement(Locator.name("content"), - "\n" + - "\n" + - " \n" + - ""); - clickButton("Import"); - - log("Publishing the data as the PI"); - - //impersonate the PI - impersonate(TEST_ASSAY_USR_PI1); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", this); - table.checkAllOnPage(); - table.clickHeaderButtonAndWait("Link to Study"); - - //the target study selected before was Study2, but the PI is not an editor there - //so ensure that system has correctly caught this fact and now asks the PI to - //select a different study, and lists only those studies in which the PI is - //an editor - - //ensure warning - assertTextPresent("WARNING: You do not have permissions to link to one or more of the selected run's associated studies."); - - //ensure that Study2 and Study 3 are not available in the target study drop down - assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2) + "']")); - assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3) + "']")); - - //Study1 is the only one left, so it should be there and already be selected - assertElementPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1) + "']")); - - // Make sure the selected study is Study1 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY1 + " Study: Verify Results"); - - setFormElement(Locator.name("visitId"), "301.5"); - clickButton("Link to Study"); - - log("Verifying that the data was published"); - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("QCState"); - _customizeViewsHelper.applyCustomView(); - assertTextPresent( - "Pending Review", - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "8", "1", "1", "1", "1", "1", "1", "2"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "8 / 9", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "2 / 3"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - doAndWaitForPageToLoad(() -> uncheckCheckbox(Locator.checkboxByNameAndValue("visitStatistic", "ParticipantCount"))); - row2 = new String[]{TEST_ASSAY, "9", "1", "1", "1", "1", "1", "1", "3"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - clickAndWait(Locator.linkWithText("9")); - - assertElementPresent(Locator.linkWithText("999320885"), 1); - assertElementPresent(Locator.linkWithText("999320885"), 1); - assertTextPresent( - "301.0", - "9.0", - "8.0", - TEST_RUN1_COMMENTS, - TEST_RUN2_COMMENTS, - TEST_RUN1, - TEST_RUN2, - "2000-06-06", - TEST_ASSAY_RUN_PROP1, - "18"); - - // test recall - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - waitAndClickAndWait(Locator.linkWithText("view link to study history")); - - // Set a filter so that we know we're recalling SecondRun - DataRegionTable region = new DataRegionTable("query", this); - region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); - doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); - - DataRegionTable linkStudy = new DataRegionTable("Dataset", this); - linkStudy.checkAll(); - doAndWaitForPageToLoad(() -> - { - linkStudy.clickHeaderButton("Recall Rows"); - acceptAlert(); - }); - assertTextPresent("row(s) were recalled from a study to the assay: " + TEST_ASSAY); - - // Set a filter so that we know we're looking at the link event for SecondRun again - region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); - - // verify audit entry was adjusted - doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); - assertTextPresent("All rows that were previously linked in this event have been recalled"); - - stopImpersonating(); - } - - /** - * Designed to test automatic timepoint generation when linking to a date based study. - * Most tests of timepoint matching are covered by separate junit tests; however, - * this will create 1 pre-existing timepoint, and when linking data this timepoint should be - * chosen for appropriate records. - */ - @LogMethod - private void publishDataToDateBasedStudy() - { - log("Prepare visit map to check PTID counts in study navigator."); - - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY3); - - clickAndWait(Locator.linkWithText("Manage")); - clickAndWait(Locator.linkWithText("Manage Timepoints")); - clickAndWait(Locator.linkWithText("Create New Timepoint")); - setFormElement(Locator.name("label"), "Preexisting Timepoint"); - setFormElement(Locator.name("sequenceNumMin"), "50"); - setFormElement(Locator.name("sequenceNumMax"), "89"); - selectOptionByText(Locator.name("typeCode"), "Screening"); - - clickButton("Save"); - assertElementPresent(Locator.tagWithAttribute("a", "data-original-title", "edit"), 1); - - //select the Lab1 folder and view all the data for the test assay - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", getDriver()); - table.checkAll(); - table.clickHeaderButtonAndWait("Link to Study"); - - checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); - - // Make sure the selected study is Study3 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY3 + " Study: Verify Results"); - - //populate initial set of values and verify the timepoint preview column - String[] dates = new String[]{"2000-02-02", "2000-03-03", "2000-04-04", "2000-05-05", "2000-06-06", "2001-01-01", "2000-01-01", "2000-02-02", "2000-03-03"}; - int idx = 1; - for (String d : dates) - { - setFormElement(Locator.xpath("(//input[@name='date'])[" + idx + "]"), d); - idx++; - } - - setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); - - DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); - linkStudy.clickHeaderButtonAndWait("Re-Validate"); - - //validate timepoints: - assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='AAA07XMC-02'] and following-sibling::td[text()='301.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='AAA07XMC-04'] and following-sibling::td[not(text())]]")); - assertElementPresent(Locator.xpath("//td[text()='Day 90 - 95' and following-sibling::td/a[text()='AAA07XSF-02'] and following-sibling::td[not(text())]]")); - - assertElementPresent(Locator.xpath("//td[text()='Day 120 - 127' and following-sibling::td/a[text()='AssayTestControl1'] and following-sibling::td[text()='5.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 152 - 159' and following-sibling::td/a[text()='AssayTestControl2'] and following-sibling::td[text()='6.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 0 - 7' and following-sibling::td/a[text()='BAQ00051-09'] and following-sibling::td[text()='7.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='BAQ00051-08'] and following-sibling::td[text()='8.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='BAQ00051-11'] and following-sibling::td[text()='9.0']]")); - - linkStudy.clickHeaderButtonAndWait("Link to Study"); - - log("Verifying that the data was published"); - assertTextPresent( - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "9", "1", "2", "2", "1", "1", "1"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "9 / 9", "1 / 1", "2 / 2", "2 / 2", "1 / 1", "1 / 1", "1 / 1"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - log("Test that correct timepoints were created"); - - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Manage Study")); - clickAndWait(Locator.linkWithText("Manage Timepoints")); - assertTextPresent( - "Day 0 - 7", - "Day 32 - 39", - "Day 90 - 95", - "Day 120 - 127", - "Day 152 - 159"); - } - - - /** - * Designed to test automatic timepoint generation when linking to a date based study. - * Most tests of timepoint matching are covered by separate junit tests; however, - * this will create 1 pre-existing timepoint, and when linking data this timepoint should be - * chosen for appropriate records. - */ - @LogMethod - private void publishDataToVisitBasedStudy() - { - log("Prepare visit map to check PTID counts in study navigator."); - - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); - - _studyHelper.goToManageVisits().goToImportVisitMap(); - setFormElement(Locator.name("content"), - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - "" - ); - clickButton("Import"); - - //select the Lab1 folder and view all the data for the test assay - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", getDriver()); - table.checkAll(); - table.clickHeaderButtonAndWait("Link to Study"); - - checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); - - // Make sure the selected study is Study2 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY2 + " Study: Verify Results"); - - //populate initial set of values and verify the timepoint preview column - String[] visits = new String[]{"302", "33", "4", "70"}; - int idx = 1; - for (String v : visits) - { - setFormElement(Locator.xpath("(//input[@name='visitId'])[" + idx + "]"), v); - idx++; - } - - setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); - - DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); - linkStudy.clickHeaderButtonAndWait("Re-Validate"); - - //validate timepoints: - assertElementPresent(Locator.xpath("//td[text()='Test Visit3' and following-sibling::td/a[text()='AAA07XMC-02']]")); - assertElementPresent(Locator.xpath("//td[text()='33.0' and following-sibling::td/a[text()='AAA07XMC-04']]")); - assertElementPresent(Locator.xpath("//td[text()='4.0' and following-sibling::td/a[text()='AAA07XSF-02']]")); - - assertElementPresent(Locator.xpath("//td[text()='Test Visit2' and following-sibling::td/a[text()='AssayTestControl1']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='AssayTestControl2']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-09']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-08']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-11']]")); - - linkStudy.clickHeaderButtonAndWait("Link to Study"); - - log("Verifying that the data was published"); - assertTextPresent( - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "9", " ", " ", " ", "1", " ", " ", "1", " ", " ", "4", " ", " ", " ", " ", "1", "1", " ", " ", " ", "1", " ", " ", " ", " ", " "}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "9 / 9", " ", " ", " ", "1 / 1", " ", " ", "1 / 1", " ", " ", "4 / 4", " ", " ", " ", " ", "1 / 1", "1 / 1", " ", " ", " ", "1 / 1", " ", " ", " ", " ", " "}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - log("Test that correct timepoints were created"); - - clickTab("Overview"); - _studyHelper.goToManageVisits(); - assertTextPresent( - "Test Visit1", - "6.0 - 13.0", - "Test Visit2", - "50.0 - 70.0", - "Test Visit3", - "302.0 - 303.0"); - } - - /** - * Tests editing of an existing assay definition - */ - @LogMethod - private void editAssay() - { - log("Testing edit and delete and assay definition"); - clickProject(getProjectName()); - waitAndClickAndWait(Locator.linkWithText(TEST_ASSAY)); - - // change a field name and label and remove a field - ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); - DomainFormPanel domainFormPanel = designerPage.expandFieldsPanel("Results"); - domainFormPanel.getField(5).setName(TEST_ASSAY_DATA_PROP_NAME + "edit"); - domainFormPanel.getField(5).setLabel(TEST_ASSAY_DATA_PROP_NAME + "edit"); - domainFormPanel.removeField(domainFormPanel.getField(4).getName(), true); - designerPage.clickFinish(); - - //ensure that label has changed in run data in Lab 1 folder - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - assertTextPresent(TEST_ASSAY_DATA_PROP_NAME + "edit"); - assertTextNotPresent(TEST_ASSAY_DATA_PROP_NAME + 4); - - AuditLogTest.verifyAuditEvent(this, AuditLogTest.ASSAY_AUDIT_EVENT, AuditLogTest.COMMENT_COLUMN, "were linked to a study from the assay: " + TEST_ASSAY, 5); - } - - @LogMethod - private void viewCrossFolderData() - { - log("Testing cross-folder data"); - - clickProject(getProjectName()); - - portalHelper.addWebPart("Assay Runs"); - selectOptionByText(Locator.name("viewProtocolId"), "General: " + TEST_ASSAY); - // assay runs has a details page that needs to be submitted - clickButton("Submit", defaultWaitForPage); - - // Set the container filter to include subfolders - DataRegionTable assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - assayRuns.setContainerFilter(DataRegionTable.ContainerFilterType.CURRENT_AND_SUBFOLDERS); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - log("Save the customized view to include subfolders"); - assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - CustomizeView customizeViewsHelper = assayRuns.getCustomizeView(); - customizeViewsHelper.openCustomizeViewPanel(); - customizeViewsHelper.saveCustomView(""); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - log("Testing select all data and view"); - assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - assayRuns.checkAllOnPage(); - clickButton("Show Results", defaultWaitForPage); - verifySpecimensPresent(3, 2, 3); - - log("Testing clicking on a run"); - clickProject(getProjectName()); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - verifySpecimensPresent(3, 2, 0); - - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable region = new DataRegionTable("Data", this); - region.clearAllFilters("SpecimenID"); - verifySpecimensPresent(3, 2, 3); - - log("Testing assay-study linkage"); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); - portalHelper.addWebPart("Datasets"); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickButton("View Source Assay", defaultWaitForPage); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - clickAndWait(Locator.linkWithText(TEST_RUN1)); - verifySpecimensPresent(3, 2, 0); - - clickAndWait(Locator.linkWithText("view results")); - region = new DataRegionTable("Data", this); - region.clearAllFilters("SpecimenID"); - verifySpecimensPresent(3, 2, 3); - - // Verify that the correct linked to study column is present - assertTextPresent("Linked to Study 1 Study"); - - log("Testing link to study availability"); - clickProject(getProjectName()); - clickAndWait(Locator.linkWithText(TEST_RUN3)); - - region = new DataRegionTable("Data", this); - region.checkAll(); - region.clickHeaderButtonAndWait("Link to Study"); - clickButton("Next"); - - verifySpecimensPresent(0, 0, 3); - - clickButton("Cancel"); - } - - @LogMethod - private void verifyStudyList() - { - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); - portalHelper.addWebPart("Study List"); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY2 + " Study")); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY3 + " Study")); - portalHelper.clickWebpartMenuItem("Studies", "Customize"); - - //verify grid view - selectOptionByText(Locator.name("displayType"), "Grid"); - clickButton("Submit"); - assertElementNotPresent(Locator.linkWithText("edit")); - - //edit study properties - clickAndWait(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); - click(Locator.tagWithAttribute("span", "title", "Edit")); - waitForElement(Locator.name("Investigator"), WAIT_FOR_JAVASCRIPT); - setFormElement(Locator.name("Investigator"), INVESTIGATOR); - setFormElement(Locator.name("Grant"), GRANT); - setFormElement(Locator.name("Description"), DESCRIPTION); - clickButton("Submit"); - - //verify study properties (grid view) - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); - DataRegionTable table = new DataRegionTable("qwpStudies", this); - assertEquals("Studies not sorted correctly.", TEST_ASSAY_FLDR_STUDY1 + " Study", table.getDataAsText(0, "Label")); - assertEquals("Failed to set study investigator.", INVESTIGATOR, table.getDataAsText(0, "Investigator")); - assertEquals("Failed to set study grant.", GRANT, table.getDataAsText(0, "Grant")); - assertEquals("Failed to set study description.", DESCRIPTION, table.getDataAsText(0, "Description")); - - //verify study properties (details view) - portalHelper.clickWebpartMenuItem("Studies", "Customize"); - selectOptionByText(Locator.name("displayType"), "Details"); - clickButton("Submit"); - assertTextPresent(INVESTIGATOR, DESCRIPTION); - assertTextNotPresent(GRANT, TEST_ASSAY_FLDR_STUDY1 + " Study tracks data"); //Old description - } - - private void verifySpecimensPresent(int aaa07Count, int controlCount, int baq00051Count) - { - // need to double the count, once for the label and once for the param in the link url - assertTextPresent("AAA07", aaa07Count * 2); - assertTextPresent("AssayTestControl", controlCount * 2); - assertTextPresent("BAQ00051", baq00051Count * 2); - } - - @Test // Issue 53625 - public void testAssayLookupValidatorConversion() - { - _containerHelper.createProject(ISSUE_53625_PROJECT, "Assay"); - goToProjectHome(ISSUE_53625_PROJECT); - - log("Create a list with an int key and a string value"); - String lookToListName = TestDataGenerator.randomDomainName("lookToList", DomainUtils.DomainKind.IntList); - String keyName = TestDataGenerator.randomFieldName("key", null, DomainUtils.DomainKind.IntList); - FieldInfo valueField = FieldInfo.random("value", FieldDefinition.ColumnType.String, DomainUtils.DomainKind.IntList); - _listHelper.createList(ISSUE_53625_PROJECT, lookToListName, keyName, valueField.getFieldDefinition()); - _listHelper.bulkImportData(TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(valueField.getName(), "One"), - Map.of(valueField.getName(), "Two"), - Map.of(valueField.getName(), "123"), - // GitHub Issue #443: value is the primary key for another row - Map.of(valueField.getName(), "5"), // pk = 4 - Map.of(valueField.getName(), "6") // pk = 5 - ), List.of(valueField.getName()), true)); - - log("Create an assay with a results lookup field to the list, with lookup validator set"); - goToProjectHome(ISSUE_53625_PROJECT); - FieldInfo lookupField = new FieldInfo("lookup", new FieldDefinition.IntLookup(null, "lists", lookToListName)); - ReactAssayDesignerPage designerPage = _assayHelper.createAssayDesign("General", ISSUE_53625_ASSAY); - designerPage.goToBatchFields() - .removeAllFields(false); - designerPage.goToResultsFields() - .removeAllFields(false) - .manuallyDefineFields(lookupField.getFieldDefinition().setLookupValidatorEnabled(true)); - designerPage.clickFinish(); - - log("Verify importing an assay run with valid and invalid values for the lookup field"); - verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithLookupValidator", true); - - log("Turn off lookup field validator and test the imports again"); - designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.goToResultsFields() - .getField(lookupField.getName()) - .setLookupValidatorEnabled(false); - designerPage.clickFinish(); - verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithoutLookupValidator", false); - - log("GitHub Issue #443: Verify that importing a value that is also a primary key maps to the titleColumn value"); - verifyAssayImportForPKValueThatIsTitleColumn(ISSUE_53625_ASSAY, lookupField, "RunWithPKandTitleColumn"); - } - - private void verifyAssayImportForPKValueThatIsTitleColumn(String assayName, FieldInfo lookupField, String runName) - { - String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "4"), // pk 4, value 5 - Map.of(lookupField.getName(), "5"), // pk 4, value 5 - Map.of(lookupField.getName(), "6")), // pk 5, value 6 - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); - checker().fatal().verifyEquals("Lookup values not as expected.", List.of("5", "5", "6"), dataTable.getColumnDataAsText(lookupField.getLabel())); - } - - private void verifyAssayImportForLookupValidator(String assayName, FieldInfo lookupField, String runName, boolean validatorOn) - { - String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "One"), // valid - Map.of(lookupField.getName(), "99")), // invalid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - assertTextPresent("Could not translate value: 99"); - if (validatorOn) assertTextPresent("Value '99' was not present in lookup target"); - else assertTextNotPresent("Value '99' was not present in lookup target"); - clickButton("Cancel"); - - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "Three"), // invalid - Map.of(lookupField.getName(), "2")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - assertTextPresent("Failed to convert"); - assertTextPresent("Could not translate value: Three"); - clickButton("Cancel"); - - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "One"), // valid - Map.of(lookupField.getName(), "2"), // valid - Map.of(lookupField.getName(), "123")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); - checker().verifyEquals("Lookup values not as expected.", List.of("One", "Two", "123"), dataTable.getColumnDataAsText(lookupField.getLabel())); - - // test with just the numeric value since that was causing issues during manual testing - runName = runName + "NumericOnly"; - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "123")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 1, dataTable.getDataRowCount()); - checker().verifyEquals("Lookup values not as expected.", List.of("123"), dataTable.getColumnDataAsText(lookupField.getLabel())); - } - - @Override - protected BrowserType bestBrowser() - { - return BrowserType.CHROME; - } -} +/* + * Copyright (c) 2016-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.study; + +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.assay.AssayListCommand; +import org.labkey.remoteapi.assay.AssayListResponse; +import org.labkey.test.Locator; +import org.labkey.test.TestFileUtils; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.WebTestHelper; +import org.labkey.test.categories.Assays; +import org.labkey.test.categories.Daily; +import org.labkey.test.components.CustomizeView; +import org.labkey.test.components.assay.AssayConstants; +import org.labkey.test.components.domain.DomainFieldRow; +import org.labkey.test.components.domain.DomainFormPanel; +import org.labkey.test.pages.ReactAssayDesignerPage; +import org.labkey.test.pages.assay.AssayBeginPage; +import org.labkey.test.pages.assay.AssayImportPage; +import org.labkey.test.pages.assay.AssayRunsPage; +import org.labkey.test.params.FieldDefinition; +import org.labkey.test.params.FieldInfo; +import org.labkey.test.params.assay.GeneralAssayDesign; +import org.labkey.test.params.experiment.SampleTypeDefinition; +import org.labkey.test.tests.AbstractAssayTest; +import org.labkey.test.tests.AuditLogTest; +import org.labkey.test.util.AuditLogHelper; +import org.labkey.test.util.DataRegionTable; +import org.labkey.test.util.DomainUtils; +import org.labkey.test.util.LogMethod; +import org.labkey.test.util.OptionalFeatureHelper; +import org.labkey.test.util.SampleTypeHelper; +import org.labkey.test.util.StudyHelper; +import org.labkey.test.util.TestDataGenerator; +import org.labkey.test.util.data.TestArrayDataUtils; +import org.labkey.test.util.data.TestDataUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.labkey.test.util.TestDataGenerator.randomTextChoice; +import static org.labkey.test.util.TestDataGenerator.shuffleSelect; +import static org.labkey.test.util.data.TestArrayDataUtils.formatMultiValueText; + +@Category({Daily.class, Assays.class}) +public class AssayTest extends AbstractAssayTest +{ + private static final String INVESTIGATOR = "Dr. No"; + private static final String GRANT = "SPECTRE"; + private static final String DESCRIPTION = "World Domination."; + private static final String ISSUE_53625_ASSAY = TestDataGenerator.randomDomainName("Issue53625", DomainUtils.DomainKind.Assay); + private static final String ISSUE_53625_PROJECT = "Issue53625Project"; + private static final String ISSUE_53616_ASSAY = "Issue53616Assay"; + private static final String ISSUE_53616_PROJECT = "Issue53616Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String ISSUE_53831_PROJECT = "Issue53831Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String SAMPLE_FIELD_TEST_ASSAY = "SampleFieldTestAssay"; + private static final String SAMPLE_FIELD_PROJECT_NAME = "Sample Field Test Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String MVTC_MULTI_FILE_IMPORT_ASSAY = TestDataGenerator.randomDomainName("MVTCMultiFileImportAssay", DomainUtils.DomainKind.Assay); + private static final String MVTC_MULTI_FILE_IMPORT_PROJECT = "MVTCMultiFileImportAssay" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String COL_ASSAY_ID_LABEL = "Assay ID"; + private static final List TEXT_MULTI_CHOICE_LIST = randomTextChoice(10); + private static final FieldInfo COL_MULTITEXTCHOICE = FieldInfo.random("Multi Choice", FieldDefinition.ColumnType.MultiValueTextChoice) + .customizeFieldDefinition(fd -> fd.setMultiChoiceValues(TEXT_MULTI_CHOICE_LIST)); + + + @Override + protected String getProjectName() + { + return TEST_ASSAY_PRJ_SECURITY; + } + + /** + * Cleanup entry point. + */ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + //should also delete the groups + _containerHelper.deleteProject(getProjectName(), false); + _containerHelper.deleteProject(SAMPLE_FIELD_PROJECT_NAME, false); + _containerHelper.deleteProject(ISSUE_53616_PROJECT, false); + _containerHelper.deleteProject(ISSUE_53625_PROJECT, false); + _containerHelper.deleteProject(ISSUE_53831_PROJECT, false); + _containerHelper.deleteProject(MVTC_MULTI_FILE_IMPORT_PROJECT, false); + + _userHelper.deleteUsers(false, TEST_ASSAY_USR_PI1, TEST_ASSAY_USR_TECH1); + } + + // Issue 53831: Assay name max length check + @Test + public void testAssayNameMaxLength() throws Exception + { + _containerHelper.createProject(ISSUE_53831_PROJECT, "Assay"); + goToProjectHome(ISSUE_53831_PROJECT); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", "a" + "0123456789".repeat(15)); + List errors = assayDesignerPage.clickSaveExpectingErrors(); + checker().verifyEquals("Wrong number of errors", 1, errors.size()); + checker().verifyEquals("Wrong error message: " + errors.get(0), + "Value is too long for assay design name, a maximum length of 150 is allowed. The supplied value, 'a01234567890123456789012...78901234567890123456789', was 151 characters long.", + errors.get(0)); + assayDesignerPage.clickCancel(); + } + + @Test + public void testAssayMultiFileImportForMVTC() throws Exception + { + Assume.assumeTrue("Multi-choice text fields are only supported on PostgreSQL", WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL); + _containerHelper.createProject(MVTC_MULTI_FILE_IMPORT_PROJECT, "Assay"); + new GeneralAssayDesign(MVTC_MULTI_FILE_IMPORT_ASSAY) + .setRunFields(List.of(new FieldDefinition("runText", FieldDefinition.ColumnType.String)), true) + .setDataFields(List.of(COL_MULTITEXTCHOICE.getFieldDefinition()), false) + .createAssay(MVTC_MULTI_FILE_IMPORT_PROJECT, createDefaultConnection()); + + String firstFileName = "MVTCAssayImport.tsv"; + String secondFileName = "MVTCAssayImportSecond.tsv"; + List> fileDataFirstImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) + .limit(5) + .toList(); + List> fileDataSecondImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) + .limit(5) + .toList(); + + log("Import first and second runs with MVTC data from files"); + AssayImportPage assayImportPage = goToManageAssays() + .clickAssay(MVTC_MULTI_FILE_IMPORT_ASSAY) + .clickImportData(); + assayImportPage.clickNext(); + assayImportPage.setDataFile(writeMultiValueFileForAssayRun(firstFileName, fileDataFirstImport)); + + assayImportPage = assayImportPage.clickSaveAndImportAnother(); + assayImportPage.setDataFile(writeMultiValueFileForAssayRun(secondFileName, fileDataSecondImport)); + assayImportPage.clickSaveAndFinish(); + + AssayRunsPage assayRunsPage = new AssayRunsPage(getDriver()); + checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.getTable().getColumnDataAsText(COL_ASSAY_ID_LABEL)) + .as("expect both runs to appear in the runs list") + .containsExactlyInAnyOrder(firstFileName, secondFileName)); + + List expectedValues = Stream.concat(fileDataFirstImport.stream(), fileDataSecondImport.stream()) + .map(values -> TestArrayDataUtils.sortAndJoin(values, " ")) + .toList(); + checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.clickViewResults().getDataTable().getColumnDataAsText(COL_MULTITEXTCHOICE)) + .as("expect MVTC values to match imported data") + .containsExactlyInAnyOrderElementsOf(expectedValues)); + } + + private File writeMultiValueFileForAssayRun(String fileName, List> fileData) throws IOException + { + List> rows = Stream.concat( + Stream.of(List.of(COL_MULTITEXTCHOICE.getName())), + fileData.stream().map(row -> List.of(formatMultiValueText(row))) + ).toList(); + return TestDataUtils.writeRowsToFile(fileName, rows); + } + + // Issue 53616: Assay creation attempt after an error results in "Assay protocol already exists for this name." + @Test + public void testFailedCreation() throws Exception + { + _containerHelper.createProject(ISSUE_53616_PROJECT, "Assay"); + goToProjectHome(ISSUE_53616_PROJECT); + + log("Create test assay"); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", ISSUE_53616_ASSAY) + .setDescription(TEST_ASSAY_DESC); + + DomainFormPanel resultsPanel = assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset result fields + resultsPanel.addField("TooLongFieldName".repeat(20)); + + log("Save initial assay design with sample field set to 'All Samples'"); + List errors = assayDesignerPage.clickSaveExpectingErrors(); + assertEquals("Wrong number of errors", 1, errors.size()); + assertTrue("Wrong error message: " + errors.get(0), errors.get(0).startsWith("Name cannot exceed 200 characters, but was")); + + resultsPanel.removeAllFields(false); + resultsPanel.addField("ShortAndSweet"); + assayDesignerPage.clickFinish(); + + AssayListCommand command = new AssayListCommand(); + AssayListResponse response = command.execute(createDefaultConnection(), ISSUE_53616_PROJECT); + assertNotNull("Didn't find expected assay design", response.getDefinition(ISSUE_53616_ASSAY)); + } + + /** + * Performs the Assay security test + * This test creates a project with a folder hierarchy with multiple groups and users; + * defines an Assay at the project level; uploads run data as a labtech; publishes + * as a PI, and tests to make sure that security is properly enforced + */ + @Test + public void testAssaySecurity() throws Exception + { + log("Starting Assay security scenario tests"); + setupEnvironment(); + setupPipeline(getProjectName()); + SpecimenImporter importer = new SpecimenImporter(TestFileUtils.getTestTempDir(), StudyHelper.SPECIMEN_ARCHIVE_A, FileUtil.appendName(TestFileUtils.getTestTempDir(), "specimensSubDir"), TEST_ASSAY_FLDR_STUDY2, 1); + importer.importAndWaitForComplete(); + defineAssay(); + uploadRuns(TEST_ASSAY_FLDR_LAB1, TEST_ASSAY_USR_TECH1); + editResults(); + publishData(); + publishDataToDateBasedStudy(); + publishDataToVisitBasedStudy(); + editAssay(); + viewCrossFolderData(); + verifyStudyList(); + verifyRunDeletionRecallsDatasetRows(); + verifyWebdavTree(); + } + + @Test + public void testSampleFieldUpdate() + { + log("Starting sample field update test"); + _containerHelper.createProject(SAMPLE_FIELD_PROJECT_NAME, "Assay"); + + log("Create test assay"); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", SAMPLE_FIELD_TEST_ASSAY) + .setDescription(TEST_ASSAY_DESC); + + assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset batch fields + + DomainFormPanel resultsPanel = assayDesignerPage.goToResultsFields().removeAllFields(false); //remove preset result fields + + String sampleFieldName = "SampleField"; + resultsPanel.manuallyDefineFields(sampleFieldName) + .setType(FieldDefinition.ColumnType.Sample) + .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); + + log("Save initial assay design with sample field set to 'All Samples'"); + assayDesignerPage.clickFinish(); + + log("Verify save successful"); + assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); + AssayBeginPage assayPage = goToManageAssays(); + assertElementPresent(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + + log("Create new Sample Types to verify against"); + String targetTypeName = "Target Sample Type"; + SampleTypeDefinition targetDefinition = new SampleTypeDefinition(targetTypeName).setFields(new ArrayList<>()); + SampleTypeHelper ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); + ssHelper.createSampleType(targetDefinition, "Name\nS_1\nS_2\nS_3"); + + String otherTypeName = "Other Sample Type"; + SampleTypeDefinition otherDefinition = new SampleTypeDefinition(otherTypeName).setFields(new ArrayList<>()); + ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); + ssHelper.createSampleType(otherDefinition, "Name\nOS_1\nOS_2"); + + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN1, "SampleField\nOS_1"); + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); + + log("Edit assay design and change Sample field to point to created Sample Type"); + goToManageAssays(); + clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.expandFieldsPanel("Results") + .getField(sampleFieldName) + .setSampleType(targetTypeName); + designerPage.clickFinish(); + + log("Verify updates saved successfully"); + assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN2, "SampleField\nS_1"); + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable table = new DataRegionTable("Data", getDriver()); + List sampleFieldValues = table.getColumnDataAsText("SampleField"); + assertTrue("First sample should not resolve to sample type", sampleFieldValues.get(0).startsWith("<")); + assertEquals("Second sample should resolve to sample type", "S_1", sampleFieldValues.get(1)); + assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); + + log("GitHub Issue #688: verify sample lookup to createdBy"); + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("SampleField/CreatedBy"); + _customizeViewsHelper.applyCustomView(); + table = new DataRegionTable("Data", getDriver()); + List createdByValues = table.getColumnDataAsText("SampleField/CreatedBy"); + assertEquals("First sample should not have a createdBy since it doesn't resolve", " ", createdByValues.get(0)); + assertEquals("Second sample should have a createdBy since it resolves to a sample type", getCurrentUserName(), createdByValues.get(1)); + + log("Edit assay design and change Sample field to point back to 'All Samples'"); + goToManageAssays(); + clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.expandFieldsPanel("Results") + .getField(sampleFieldName) + .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); + designerPage.clickFinish(); + assertEquals("Error saving updated sample field", 0, checker().errorsSinceMark()); + + log("Verify updates saved successfully"); + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN3, "SampleField\nS_2\nOS_2"); + assertEquals("Error importing data after assay sample field update", 0, checker().errorsSinceMark()); + + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); + assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); + assertElementPresent("Sample lookup failed for: S_2", new Locator.LinkLocator("S_2"), 1); + assertElementPresent("Sample lookup failed for: OS_2", new Locator.LinkLocator("OS_2"), 1); + + log("GitHub Issue #688: verify sample lookup to createdBy"); + table = new DataRegionTable("Data", getDriver()); + for (int i = 0; i < table.getDataRowCount(); i++) + assertEquals("Row " + i + " should have current user as createdBy since they all resolve to samples", getCurrentUserName(), table.getDataAsText(i, "SampleField/CreatedBy")); + } + + private void importAssayData(String assayName, String runName, String runDataStr) + { + goToManageAssays(); + clickAndWait(Locator.linkWithText(assayName)); + clickButton("Import Data", "Run Data"); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, runName); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, runDataStr); + clickButton("Save and Finish"); + + } + + @LogMethod + private void verifyRunDeletionRecallsDatasetRows() + { + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + DataRegionTable assayRuns = new DataRegionTable("Runs", this); + assayRuns.checkCheckbox(0); + assayRuns.clickHeaderButtonAndWait("Delete"); + // Make sure that it shows that the data is part of study datasets + assertTextPresent(TEST_RUN3, "2 dataset(s)", TEST_ASSAY); + assertTextNotPresent("FirstRun"); + // Do the delete + clickButton("Confirm Delete"); + + // Be sure that we have a special audit record + clickAndWait(Locator.linkWithText("view link to study history")); + assertTextPresent("3 row(s) were recalled from a study to the assay: "); + + // Verify that the deleted run data is gone from the dataset + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); + clickAndWait(Locator.linkWithText("1 dataset")); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + assertTextPresent("AAA07XMC-04", TEST_RUN1); + assertTextNotPresent("BAQ00051-09", TEST_RUN3); + } + + //Issue 12203: Incorrect files are visible from pipeline directory + private void verifyWebdavTree() + { + beginAt("_webdav"); + _fileBrowserHelper.selectFileBrowserItem(getProjectName() + "/Studies/Study 1/"); + Locator.XPathLocator l = Locator.xpath("//span[text()='@pipeline']"); + assertElementPresent(l, 1); + } + + @LogMethod + private void editResults() throws IOException, CommandException + { + // Verify that the results aren't editable by default + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + waitAndClickAndWait(Locator.linkWithText("view results")); + DataRegionTable table = new DataRegionTable("Data", getDriver()); + assertEquals("No rows should be editable", 0, DataRegionTable.updateLinkLocator().findElements(table.getComponentElement()).size()); + assertElementNotPresent(Locator.button("Delete")); + + // Edit the design to make them editable + ReactAssayDesignerPage assayDesignerPage = _assayHelper.clickEditAssayDesign(true); + assayDesignerPage.setEditableResults(true); + assayDesignerPage.clickFinish(); + + // Try an edit + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + assertEquals("Incorrect number of results shown.", 10, table.getDataRowCount()); + doAndWaitForPageToLoad(() -> dataTable.updateLink(dataTable.getRowIndex("Specimen ID", "AAA07XK5-05")).click()); + setFormElement(Locator.name("quf_SpecimenID"), "EditedSpecimenID"); + setFormElement(Locator.name("quf_VisitID"), "601.5"); + setFormElement(Locator.name("quf_testAssayDataProp5"), "notAnumber"); + clickButton("Submit"); + assertTextPresent("Could not convert value: " + "notAnumber"); + setFormElement(Locator.name("quf_testAssayDataProp5"), "514801"); + clickButton("Submit"); + assertTextPresent("EditedSpecimenID", "601.5", "514801"); + + // Try a delete + dataTable.checkCheckbox(table.getRowIndex("Specimen ID", "EditedSpecimenID")); + doAndWaitForPageToLoad(() -> + { + dataTable.clickHeaderButton("Delete"); + assertAlert("Are you sure you want to delete the selected row?"); + }); + + // Verify that the edit was audited + AuditLogHelper auditLogHelper = new AuditLogHelper(this, () -> WebTestHelper.getRemoteApiConnection(false)); + auditLogHelper.checkAuditEventDiffCount(getProjectName(), AuditLogHelper.AuditEvent.QUERY_UPDATE_AUDIT_EVENT, List.of(0/*delete*/, 3/*edit*/)); + + goToSchemaBrowser(); + viewQueryData("auditLog", "ExperimentAuditEvent"); + assertTextPresent("1 data row has been edited in " + TEST_ASSAY + "."); + + } + + /** + * Generates the text that appears in the target study drop-down for a given study name + * @param studyName name of the target study + * @return formatted string of what appears in the target study drop-down + */ + private String getTargetStudyOptionText(String studyName) + { + //the format used in the drop down is: + // /// ( Study) + return "/" + getProjectName() + "/" + TEST_ASSAY_FLDR_STUDIES + "/" + + studyName + " (" + studyName + " Study)"; + } + + /** + * Uploads run data for the centrally defined Assay while impersonating a labtech-style user + * @param folder name of the folder into which we should upload + * @param asUser the user to impersonate before uploading + */ + @LogMethod + private void uploadRuns(String folder, String asUser) + { + log("Uploading runs into folder " + folder + " as user " + asUser); + navigateToFolder(getProjectName(), folder); + impersonate(asUser); + + clickAndWait(Locator.linkWithText("Assay List")); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + + //nav trail check + assertNavTrail("Assay List", TEST_ASSAY + " Batches"); + + clickButton("Import Data"); + assertTextPresent(TEST_ASSAY_SET_PROP_NAME + "3"); + + log("Batch properties"); + clickButton("Next"); + assertTextPresent(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + " is required and must be of type Number (Double)."); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), "Bad Test"); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), "Bad Test"); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), "Bad Test"); + clickButton("Next"); + assertTextPresent( + "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.", + "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.", + "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY)."); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]); + + //ensure that the target study drop down contains Study 1 and Study 2 only and not Study 3 + //(labtech1 does not have read perms to Study 3) + waitForElement(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1))); + assertElementPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2))); + assertElementNotPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3))); + + //select Study2 as the target study (note that PI is not an Editor in this study so we can test for override case) + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); + + clickButton("Next"); + + log("Check properties set."); + assertTextPresent( + TEST_ASSAY_SET_PROPERTIES[1], + TEST_ASSAY_SET_PROPERTIES[2], + TEST_ASSAY_SET_PROPERTIES[3], + TEST_ASSAY_SET_PROPERTIES[0]); + + log("Run properties and data"); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC); + clickButton("Save and Finish"); + + assertTextPresent(TEST_ASSAY_RUN_PROP_NAME + "0 is required and must be of type Text (String)."); + assertTextPresent(PROTOCOL_DOC.getName()); + waitAndClick(Locator.linkWithText("remove")); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN1); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN1_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); + clickButton("Save and Finish"); + + Locator loc4 = Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"); + assertEquals("", getFormElement(loc4)); + assertTextPresent("Data file contained zero data rows"); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN1_DATA1); + clickButton("Save and Import Another Run"); + + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN2); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN2_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA1); + clickButton("Save and Finish"); + + assertTextPresent(PROTOCOL_DOC2.getName()); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA2); + clickButton("Save and Finish"); + + assertTextPresent("Could not convert value 'g' (String) for Double field 'VisitID'"); + assertTextPresent(PROTOCOL_DOC2.getName()); + assertEquals(TEST_RUN2, getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); + assertEquals(TEST_RUN2_COMMENTS, getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA3); + clickButton("Save and Import Another Run"); + + assertTextPresent("Missing value for required property: " + TEST_ASSAY_DATA_PROP_NAME + "6"); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA4); + clickButton("Save and Import Another Run"); + + assertEquals("", getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); + assertEquals("", getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN3); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN3_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); + clickButton("Save and Finish"); + + assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN3_DATA1); + clickButton("Save and Finish"); + + // Verify the first run did not have a file, the second run had the attached file and the third run had a file + // with a unique name. + assertTextNotPresent(PROTOCOL_DOC.getName()); + assertTextPresent(PROTOCOL_DOC2.getName()); + assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); + + log("Check out the data for one of the runs"); + assertNoLabKeyErrors(); + assertTextPresent( + TEST_ASSAY + " Runs", + TEST_ASSAY_RUN_PROP1, + TEST_ASSAY_SET_PROPERTIES[0], + TEST_ASSAY_SET_PROPERTIES[3]); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + assertElementNotPresent(Locator.tagWithText("td", "7.0")); + // Make sure that our specimen IDs resolved correctly + assertTextPresent( + "AAA07XSF-02", + "999320885", + "301", + "AAA07XK5-05", + "999320812", + "601", + TEST_ASSAY_DATA_PROP_NAME + "4", + TEST_ASSAY_DATA_PROP_NAME + "5", + TEST_ASSAY_DATA_PROP_NAME + "6", + "2000-06-06", + "0.0", + "f", + ALIASED_DATA); + + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("SpecimenID/GlobalUniqueId"); + _customizeViewsHelper.addColumn("SpecimenID/Specimen/PrimaryType"); + _customizeViewsHelper.addColumn("SpecimenID/AssayMatch"); + _customizeViewsHelper.removeColumn("Run/testAssayRunProp1"); + _customizeViewsHelper.removeColumn("Run/Batch/testAssaySetProp2"); + _customizeViewsHelper.removeColumn("testAssayDataProp4"); + _customizeViewsHelper.applyCustomView(); + + assertTextPresent("Blood (Whole)", 4); + + Locator.XPathLocator trueLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'true']"); + int totalTrues = getElementCount(trueLocator); + assertEquals(4, totalTrues); + + DataRegionTable region = new DataRegionTable("Data", this); + region.setFilter("SpecimenID", "Starts With", "AssayTestControl"); + + // verify that there are no trues showing for the assay match column that were filtered out + totalTrues = getElementCount(trueLocator); + assertEquals(0, totalTrues); + + log("Check out the data for all of the runs"); + clickAndWait(Locator.linkWithText("view results")); + region.clearAllFilters("SpecimenID"); + assertElementPresent(Locator.tagWithText("td", "7.0")); + assertElementPresent(Locator.tagWithText("td", "18")); + + assertTextPresent("Blood (Whole)", 7); + + Locator.XPathLocator falseLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'false']"); + int totalFalses = getElementCount(falseLocator); + assertEquals(3, totalFalses); + + region.setFilter("SpecimenID", "Does Not Start With", "BAQ"); + + // verify the falses have been filtered out + totalFalses = getElementCount(falseLocator); + assertEquals(0, totalFalses); + + stopImpersonating(); + } + + /** + * Impersonates the PI user and publishes the data previous uploaded. + * This will also verify that the PI cannot publish to studies for which + * the PI does not have Editor permissions. + */ + @LogMethod + private void publishData() + { + log("Prepare visit map to check PTID counts in study navigator."); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); + _studyHelper.goToManageVisits().goToImportVisitMap(); + setFormElement(Locator.name("content"), + "\n" + + "\n" + + " \n" + + ""); + clickButton("Import"); + + log("Publishing the data as the PI"); + + //impersonate the PI + impersonate(TEST_ASSAY_USR_PI1); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", this); + table.checkAllOnPage(); + table.clickHeaderButtonAndWait("Link to Study"); + + //the target study selected before was Study2, but the PI is not an editor there + //so ensure that system has correctly caught this fact and now asks the PI to + //select a different study, and lists only those studies in which the PI is + //an editor + + //ensure warning + assertTextPresent("WARNING: You do not have permissions to link to one or more of the selected run's associated studies."); + + //ensure that Study2 and Study 3 are not available in the target study drop down + assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2) + "']")); + assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3) + "']")); + + //Study1 is the only one left, so it should be there and already be selected + assertElementPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1) + "']")); + + // Make sure the selected study is Study1 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY1 + " Study: Verify Results"); + + setFormElement(Locator.name("visitId"), "301.5"); + clickButton("Link to Study"); + + log("Verifying that the data was published"); + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("QCState"); + _customizeViewsHelper.applyCustomView(); + assertTextPresent( + "Pending Review", + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "8", "1", "1", "1", "1", "1", "1", "2"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "8 / 9", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "2 / 3"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + doAndWaitForPageToLoad(() -> uncheckCheckbox(Locator.checkboxByNameAndValue("visitStatistic", "ParticipantCount"))); + row2 = new String[]{TEST_ASSAY, "9", "1", "1", "1", "1", "1", "1", "3"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + clickAndWait(Locator.linkWithText("9")); + + assertElementPresent(Locator.linkWithText("999320885"), 1); + assertElementPresent(Locator.linkWithText("999320885"), 1); + assertTextPresent( + "301.0", + "9.0", + "8.0", + TEST_RUN1_COMMENTS, + TEST_RUN2_COMMENTS, + TEST_RUN1, + TEST_RUN2, + "2000-06-06", + TEST_ASSAY_RUN_PROP1, + "18"); + + // test recall + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + waitAndClickAndWait(Locator.linkWithText("view link to study history")); + + // Set a filter so that we know we're recalling SecondRun + DataRegionTable region = new DataRegionTable("query", this); + region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); + doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); + + DataRegionTable linkStudy = new DataRegionTable("Dataset", this); + linkStudy.checkAll(); + doAndWaitForPageToLoad(() -> + { + linkStudy.clickHeaderButton("Recall Rows"); + acceptAlert(); + }); + assertTextPresent("row(s) were recalled from a study to the assay: " + TEST_ASSAY); + + // Set a filter so that we know we're looking at the link event for SecondRun again + region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); + + // verify audit entry was adjusted + doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); + assertTextPresent("All rows that were previously linked in this event have been recalled"); + + stopImpersonating(); + } + + /** + * Designed to test automatic timepoint generation when linking to a date based study. + * Most tests of timepoint matching are covered by separate junit tests; however, + * this will create 1 pre-existing timepoint, and when linking data this timepoint should be + * chosen for appropriate records. + */ + @LogMethod + private void publishDataToDateBasedStudy() + { + log("Prepare visit map to check PTID counts in study navigator."); + + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY3); + + clickAndWait(Locator.linkWithText("Manage")); + clickAndWait(Locator.linkWithText("Manage Timepoints")); + clickAndWait(Locator.linkWithText("Create New Timepoint")); + setFormElement(Locator.name("label"), "Preexisting Timepoint"); + setFormElement(Locator.name("sequenceNumMin"), "50"); + setFormElement(Locator.name("sequenceNumMax"), "89"); + selectOptionByText(Locator.name("typeCode"), "Screening"); + + clickButton("Save"); + assertElementPresent(Locator.tagWithAttribute("a", "data-original-title", "edit"), 1); + + //select the Lab1 folder and view all the data for the test assay + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", getDriver()); + table.checkAll(); + table.clickHeaderButtonAndWait("Link to Study"); + + checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); + + // Make sure the selected study is Study3 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY3 + " Study: Verify Results"); + + //populate initial set of values and verify the timepoint preview column + String[] dates = new String[]{"2000-02-02", "2000-03-03", "2000-04-04", "2000-05-05", "2000-06-06", "2001-01-01", "2000-01-01", "2000-02-02", "2000-03-03"}; + int idx = 1; + for (String d : dates) + { + setFormElement(Locator.xpath("(//input[@name='date'])[" + idx + "]"), d); + idx++; + } + + setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); + + DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); + linkStudy.clickHeaderButtonAndWait("Re-Validate"); + + //validate timepoints: + assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='AAA07XMC-02'] and following-sibling::td[text()='301.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='AAA07XMC-04'] and following-sibling::td[not(text())]]")); + assertElementPresent(Locator.xpath("//td[text()='Day 90 - 95' and following-sibling::td/a[text()='AAA07XSF-02'] and following-sibling::td[not(text())]]")); + + assertElementPresent(Locator.xpath("//td[text()='Day 120 - 127' and following-sibling::td/a[text()='AssayTestControl1'] and following-sibling::td[text()='5.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 152 - 159' and following-sibling::td/a[text()='AssayTestControl2'] and following-sibling::td[text()='6.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 0 - 7' and following-sibling::td/a[text()='BAQ00051-09'] and following-sibling::td[text()='7.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='BAQ00051-08'] and following-sibling::td[text()='8.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='BAQ00051-11'] and following-sibling::td[text()='9.0']]")); + + linkStudy.clickHeaderButtonAndWait("Link to Study"); + + log("Verifying that the data was published"); + assertTextPresent( + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "9", "1", "2", "2", "1", "1", "1"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "9 / 9", "1 / 1", "2 / 2", "2 / 2", "1 / 1", "1 / 1", "1 / 1"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + log("Test that correct timepoints were created"); + + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Manage Study")); + clickAndWait(Locator.linkWithText("Manage Timepoints")); + assertTextPresent( + "Day 0 - 7", + "Day 32 - 39", + "Day 90 - 95", + "Day 120 - 127", + "Day 152 - 159"); + } + + + /** + * Designed to test automatic timepoint generation when linking to a date based study. + * Most tests of timepoint matching are covered by separate junit tests; however, + * this will create 1 pre-existing timepoint, and when linking data this timepoint should be + * chosen for appropriate records. + */ + @LogMethod + private void publishDataToVisitBasedStudy() + { + log("Prepare visit map to check PTID counts in study navigator."); + + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); + + _studyHelper.goToManageVisits().goToImportVisitMap(); + setFormElement(Locator.name("content"), + "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + "" + ); + clickButton("Import"); + + //select the Lab1 folder and view all the data for the test assay + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", getDriver()); + table.checkAll(); + table.clickHeaderButtonAndWait("Link to Study"); + + checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); + + // Make sure the selected study is Study2 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY2 + " Study: Verify Results"); + + //populate initial set of values and verify the timepoint preview column + String[] visits = new String[]{"302", "33", "4", "70"}; + int idx = 1; + for (String v : visits) + { + setFormElement(Locator.xpath("(//input[@name='visitId'])[" + idx + "]"), v); + idx++; + } + + setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); + + DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); + linkStudy.clickHeaderButtonAndWait("Re-Validate"); + + //validate timepoints: + assertElementPresent(Locator.xpath("//td[text()='Test Visit3' and following-sibling::td/a[text()='AAA07XMC-02']]")); + assertElementPresent(Locator.xpath("//td[text()='33.0' and following-sibling::td/a[text()='AAA07XMC-04']]")); + assertElementPresent(Locator.xpath("//td[text()='4.0' and following-sibling::td/a[text()='AAA07XSF-02']]")); + + assertElementPresent(Locator.xpath("//td[text()='Test Visit2' and following-sibling::td/a[text()='AssayTestControl1']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='AssayTestControl2']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-09']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-08']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-11']]")); + + linkStudy.clickHeaderButtonAndWait("Link to Study"); + + log("Verifying that the data was published"); + assertTextPresent( + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "9", " ", " ", " ", "1", " ", " ", "1", " ", " ", "4", " ", " ", " ", " ", "1", "1", " ", " ", " ", "1", " ", " ", " ", " ", " "}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "9 / 9", " ", " ", " ", "1 / 1", " ", " ", "1 / 1", " ", " ", "4 / 4", " ", " ", " ", " ", "1 / 1", "1 / 1", " ", " ", " ", "1 / 1", " ", " ", " ", " ", " "}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + log("Test that correct timepoints were created"); + + clickTab("Overview"); + _studyHelper.goToManageVisits(); + assertTextPresent( + "Test Visit1", + "6.0 - 13.0", + "Test Visit2", + "50.0 - 70.0", + "Test Visit3", + "302.0 - 303.0"); + } + + /** + * Tests editing of an existing assay definition + */ + @LogMethod + private void editAssay() + { + log("Testing edit and delete and assay definition"); + clickProject(getProjectName()); + waitAndClickAndWait(Locator.linkWithText(TEST_ASSAY)); + + // change a field name and label and remove a field + ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); + DomainFormPanel domainFormPanel = designerPage.expandFieldsPanel("Results"); + domainFormPanel.getField(5).setName(TEST_ASSAY_DATA_PROP_NAME + "edit"); + domainFormPanel.getField(5).setLabel(TEST_ASSAY_DATA_PROP_NAME + "edit"); + domainFormPanel.removeField(domainFormPanel.getField(4).getName(), true); + designerPage.clickFinish(); + + //ensure that label has changed in run data in Lab 1 folder + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + assertTextPresent(TEST_ASSAY_DATA_PROP_NAME + "edit"); + assertTextNotPresent(TEST_ASSAY_DATA_PROP_NAME + 4); + + AuditLogTest.verifyAuditEvent(this, AuditLogTest.ASSAY_AUDIT_EVENT, AuditLogTest.COMMENT_COLUMN, "were linked to a study from the assay: " + TEST_ASSAY, 5); + } + + @LogMethod + private void viewCrossFolderData() + { + log("Testing cross-folder data"); + + clickProject(getProjectName()); + + portalHelper.addWebPart("Assay Runs"); + selectOptionByText(Locator.name("viewProtocolId"), "General: " + TEST_ASSAY); + // assay runs has a details page that needs to be submitted + clickButton("Submit", defaultWaitForPage); + + // Set the container filter to include subfolders + DataRegionTable assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + assayRuns.setContainerFilter(DataRegionTable.ContainerFilterType.CURRENT_AND_SUBFOLDERS); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + log("Save the customized view to include subfolders"); + assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + CustomizeView customizeViewsHelper = assayRuns.getCustomizeView(); + customizeViewsHelper.openCustomizeViewPanel(); + customizeViewsHelper.saveCustomView(""); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + log("Testing select all data and view"); + assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + assayRuns.checkAllOnPage(); + clickButton("Show Results", defaultWaitForPage); + verifySpecimensPresent(3, 2, 3); + + log("Testing clicking on a run"); + clickProject(getProjectName()); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + verifySpecimensPresent(3, 2, 0); + + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable region = new DataRegionTable("Data", this); + region.clearAllFilters("SpecimenID"); + verifySpecimensPresent(3, 2, 3); + + log("Testing assay-study linkage"); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); + portalHelper.addWebPart("Datasets"); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickButton("View Source Assay", defaultWaitForPage); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + clickAndWait(Locator.linkWithText(TEST_RUN1)); + verifySpecimensPresent(3, 2, 0); + + clickAndWait(Locator.linkWithText("view results")); + region = new DataRegionTable("Data", this); + region.clearAllFilters("SpecimenID"); + verifySpecimensPresent(3, 2, 3); + + // Verify that the correct linked to study column is present + assertTextPresent("Linked to Study 1 Study"); + + log("Testing link to study availability"); + clickProject(getProjectName()); + clickAndWait(Locator.linkWithText(TEST_RUN3)); + + region = new DataRegionTable("Data", this); + region.checkAll(); + region.clickHeaderButtonAndWait("Link to Study"); + clickButton("Next"); + + verifySpecimensPresent(0, 0, 3); + + clickButton("Cancel"); + } + + @LogMethod + private void verifyStudyList() + { + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); + portalHelper.addWebPart("Study List"); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY2 + " Study")); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY3 + " Study")); + portalHelper.clickWebpartMenuItem("Studies", "Customize"); + + //verify grid view + selectOptionByText(Locator.name("displayType"), "Grid"); + clickButton("Submit"); + assertElementNotPresent(Locator.linkWithText("edit")); + + //edit study properties + clickAndWait(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); + click(Locator.tagWithAttribute("span", "title", "Edit")); + waitForElement(Locator.name("Investigator"), WAIT_FOR_JAVASCRIPT); + setFormElement(Locator.name("Investigator"), INVESTIGATOR); + setFormElement(Locator.name("Grant"), GRANT); + setFormElement(Locator.name("Description"), DESCRIPTION); + clickButton("Submit"); + + //verify study properties (grid view) + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); + DataRegionTable table = new DataRegionTable("qwpStudies", this); + assertEquals("Studies not sorted correctly.", TEST_ASSAY_FLDR_STUDY1 + " Study", table.getDataAsText(0, "Label")); + assertEquals("Failed to set study investigator.", INVESTIGATOR, table.getDataAsText(0, "Investigator")); + assertEquals("Failed to set study grant.", GRANT, table.getDataAsText(0, "Grant")); + assertEquals("Failed to set study description.", DESCRIPTION, table.getDataAsText(0, "Description")); + + //verify study properties (details view) + portalHelper.clickWebpartMenuItem("Studies", "Customize"); + selectOptionByText(Locator.name("displayType"), "Details"); + clickButton("Submit"); + assertTextPresent(INVESTIGATOR, DESCRIPTION); + assertTextNotPresent(GRANT, TEST_ASSAY_FLDR_STUDY1 + " Study tracks data"); //Old description + } + + private void verifySpecimensPresent(int aaa07Count, int controlCount, int baq00051Count) + { + // need to double the count, once for the label and once for the param in the link url + assertTextPresent("AAA07", aaa07Count * 2); + assertTextPresent("AssayTestControl", controlCount * 2); + assertTextPresent("BAQ00051", baq00051Count * 2); + } + + @Test // Issue 53625 + public void testAssayLookupValidatorConversion() + { + _containerHelper.createProject(ISSUE_53625_PROJECT, "Assay"); + goToProjectHome(ISSUE_53625_PROJECT); + + log("Create a list with an int key and a string value"); + String lookToListName = TestDataGenerator.randomDomainName("lookToList", DomainUtils.DomainKind.IntList); + String keyName = TestDataGenerator.randomFieldName("key", null, DomainUtils.DomainKind.IntList); + FieldInfo valueField = FieldInfo.random("value", FieldDefinition.ColumnType.String, DomainUtils.DomainKind.IntList); + _listHelper.createList(ISSUE_53625_PROJECT, lookToListName, keyName, valueField.getFieldDefinition()); + _listHelper.bulkImportData(TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(valueField.getName(), "One"), + Map.of(valueField.getName(), "Two"), + Map.of(valueField.getName(), "123"), + // GitHub Issue #443: value is the primary key for another row + Map.of(valueField.getName(), "5"), // pk = 4 + Map.of(valueField.getName(), "6") // pk = 5 + ), List.of(valueField.getName()), true)); + + log("Create an assay with a results lookup field to the list, with lookup validator set"); + goToProjectHome(ISSUE_53625_PROJECT); + FieldInfo lookupField = new FieldInfo("lookup", new FieldDefinition.IntLookup(null, "lists", lookToListName)); + ReactAssayDesignerPage designerPage = _assayHelper.createAssayDesign("General", ISSUE_53625_ASSAY); + designerPage.goToBatchFields() + .removeAllFields(false); + designerPage.goToResultsFields() + .removeAllFields(false) + .manuallyDefineFields(lookupField.getFieldDefinition().setLookupValidatorEnabled(true)); + designerPage.clickFinish(); + + log("Verify importing an assay run with valid and invalid values for the lookup field"); + verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithLookupValidator", true); + + log("Turn off lookup field validator and test the imports again"); + designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.goToResultsFields() + .getField(lookupField.getName()) + .setLookupValidatorEnabled(false); + designerPage.clickFinish(); + verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithoutLookupValidator", false); + + log("GitHub Issue #443: Verify that importing a value that is also a primary key maps to the titleColumn value"); + verifyAssayImportForPKValueThatIsTitleColumn(ISSUE_53625_ASSAY, lookupField, "RunWithPKandTitleColumn"); + } + + private void verifyAssayImportForPKValueThatIsTitleColumn(String assayName, FieldInfo lookupField, String runName) + { + String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "4"), // pk 4, value 5 + Map.of(lookupField.getName(), "5"), // pk 4, value 5 + Map.of(lookupField.getName(), "6")), // pk 5, value 6 + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); + checker().fatal().verifyEquals("Lookup values not as expected.", List.of("5", "5", "6"), dataTable.getColumnDataAsText(lookupField.getLabel())); + } + + private void verifyAssayImportForLookupValidator(String assayName, FieldInfo lookupField, String runName, boolean validatorOn) + { + String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "One"), // valid + Map.of(lookupField.getName(), "99")), // invalid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + assertTextPresent("Could not translate value: 99"); + if (validatorOn) assertTextPresent("Value '99' was not present in lookup target"); + else assertTextNotPresent("Value '99' was not present in lookup target"); + clickButton("Cancel"); + + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "Three"), // invalid + Map.of(lookupField.getName(), "2")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + assertTextPresent("Failed to convert"); + assertTextPresent("Could not translate value: Three"); + clickButton("Cancel"); + + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "One"), // valid + Map.of(lookupField.getName(), "2"), // valid + Map.of(lookupField.getName(), "123")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); + checker().verifyEquals("Lookup values not as expected.", List.of("One", "Two", "123"), dataTable.getColumnDataAsText(lookupField.getLabel())); + + // test with just the numeric value since that was causing issues during manual testing + runName = runName + "NumericOnly"; + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "123")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 1, dataTable.getDataRowCount()); + checker().verifyEquals("Lookup values not as expected.", List.of("123"), dataTable.getColumnDataAsText(lookupField.getLabel())); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } +} diff --git a/study/test/src/org/labkey/test/tests/study/StudyDatasetsTest.java b/study/test/src/org/labkey/test/tests/study/StudyDatasetsTest.java index 202ceadbffa..5fee9eeeb67 100644 --- a/study/test/src/org/labkey/test/tests/study/StudyDatasetsTest.java +++ b/study/test/src/org/labkey/test/tests/study/StudyDatasetsTest.java @@ -189,7 +189,6 @@ public void testDatasets() public void testDatasetWithMultiChoice() { Assume.assumeTrue("Multi-choice text fields are only supported on PostgreSQL", WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL); - OptionalFeatureHelper.enableOptionalFeature(getCurrentTest().createDefaultConnection(), "multiChoiceDataType"); String datasetName = "Test dataset"; DatasetDesignerPage definitionPage = _studyHelper.goToManageDatasets() .clickCreateNewDataset() From ce32c9378d2d7b40d9c86fad7a8a397b2c49f2c2 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 26 Apr 2026 17:00:09 -0700 Subject: [PATCH 2/3] crlf --- .../labkey/experiment/ExperimentModule.java | 2372 ++++++++-------- .../labkey/test/tests/study/AssayTest.java | 2442 ++++++++--------- 2 files changed, 2407 insertions(+), 2407 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 966f924dbd2..51980210669 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1186 +1,1186 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AbstractAssayProvider; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.assay.transform.DataTransformService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.api.writer.ContainerUser; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.SampleTypeUpdateServiceDI; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; -import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 26.005; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false, true); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", - "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); - - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", - "Support for querying lineage of experiment objects", false, true); - OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", - "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false, true); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString() - ).getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString(), - "%\"" + DataTransformService.TransformOperation.UPDATE + "\"%" - ).getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, - "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", - AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, - ExpProtocol.Status.Active.toString(), - "%\"" + DataTransformService.TransformOperation.INSERT + "\"%" - ).getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - // metric to count the number of Luminex and Standard assay runs that were imported with > 1 data file - assayMetrics.put("assayRunsWithMultipleInputFiles", new SqlSelector(schema, """ - SELECT COUNT(*) FROM ( - SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data - WHERE lsid NOT LIKE '%:RelatedFile.%' AND sourceapplicationid IN ( - SELECT rowid FROM exp.protocolapplication - WHERE lsid LIKE '%:SimpleProtocol.CoreStep' AND (protocollsid LIKE '%:LuminexAssayProtocol.%' OR protocollsid LIKE '%:GeneralAssayProtocol.%') - ) - GROUP BY sourceapplicationid - ) x WHERE count > 1""").getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); - results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); - results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithMultiValueColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithMultiValueColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - } - - @Override - public void registerMigrationHandlers(@NotNull DatabaseMigrationService service) - { - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - service.registerSchemaHandler(handler); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - service.registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - service.registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = super.getPageContextJson(context); - json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); - return json; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AbstractAssayProvider; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.api.writer.ContainerUser; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 26.005; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false, true); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", + "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); + + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", + "Support for querying lineage of experiment objects", false, true); + OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", + "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false, true); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString() + ).getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString(), + "%\"" + DataTransformService.TransformOperation.UPDATE + "\"%" + ).getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, + "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", + AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, + ExpProtocol.Status.Active.toString(), + "%\"" + DataTransformService.TransformOperation.INSERT + "\"%" + ).getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + // metric to count the number of Luminex and Standard assay runs that were imported with > 1 data file + assayMetrics.put("assayRunsWithMultipleInputFiles", new SqlSelector(schema, """ + SELECT COUNT(*) FROM ( + SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data + WHERE lsid NOT LIKE '%:RelatedFile.%' AND sourceapplicationid IN ( + SELECT rowid FROM exp.protocolapplication + WHERE lsid LIKE '%:SimpleProtocol.CoreStep' AND (protocollsid LIKE '%:LuminexAssayProtocol.%' OR protocollsid LIKE '%:GeneralAssayProtocol.%') + ) + GROUP BY sourceapplicationid + ) x WHERE count > 1""").getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); + results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithMultiValueColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithMultiValueColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + } + + @Override + public void registerMigrationHandlers(@NotNull DatabaseMigrationService service) + { + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + service.registerSchemaHandler(handler); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + service.registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + service.registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = super.getPageContextJson(context); + json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); + return json; + } +} diff --git a/study/test/src/org/labkey/test/tests/study/AssayTest.java b/study/test/src/org/labkey/test/tests/study/AssayTest.java index ab4555ddf0c..02f190109e7 100644 --- a/study/test/src/org/labkey/test/tests/study/AssayTest.java +++ b/study/test/src/org/labkey/test/tests/study/AssayTest.java @@ -1,1221 +1,1221 @@ -/* - * Copyright (c) 2016-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.test.tests.study; - -import org.assertj.core.api.Assertions; -import org.junit.Assume; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.labkey.api.util.FileUtil; -import org.labkey.remoteapi.CommandException; -import org.labkey.remoteapi.assay.AssayListCommand; -import org.labkey.remoteapi.assay.AssayListResponse; -import org.labkey.test.Locator; -import org.labkey.test.TestFileUtils; -import org.labkey.test.TestTimeoutException; -import org.labkey.test.WebTestHelper; -import org.labkey.test.categories.Assays; -import org.labkey.test.categories.Daily; -import org.labkey.test.components.CustomizeView; -import org.labkey.test.components.assay.AssayConstants; -import org.labkey.test.components.domain.DomainFieldRow; -import org.labkey.test.components.domain.DomainFormPanel; -import org.labkey.test.pages.ReactAssayDesignerPage; -import org.labkey.test.pages.assay.AssayBeginPage; -import org.labkey.test.pages.assay.AssayImportPage; -import org.labkey.test.pages.assay.AssayRunsPage; -import org.labkey.test.params.FieldDefinition; -import org.labkey.test.params.FieldInfo; -import org.labkey.test.params.assay.GeneralAssayDesign; -import org.labkey.test.params.experiment.SampleTypeDefinition; -import org.labkey.test.tests.AbstractAssayTest; -import org.labkey.test.tests.AuditLogTest; -import org.labkey.test.util.AuditLogHelper; -import org.labkey.test.util.DataRegionTable; -import org.labkey.test.util.DomainUtils; -import org.labkey.test.util.LogMethod; -import org.labkey.test.util.OptionalFeatureHelper; -import org.labkey.test.util.SampleTypeHelper; -import org.labkey.test.util.StudyHelper; -import org.labkey.test.util.TestDataGenerator; -import org.labkey.test.util.data.TestArrayDataUtils; -import org.labkey.test.util.data.TestDataUtils; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.labkey.test.util.TestDataGenerator.randomTextChoice; -import static org.labkey.test.util.TestDataGenerator.shuffleSelect; -import static org.labkey.test.util.data.TestArrayDataUtils.formatMultiValueText; - -@Category({Daily.class, Assays.class}) -public class AssayTest extends AbstractAssayTest -{ - private static final String INVESTIGATOR = "Dr. No"; - private static final String GRANT = "SPECTRE"; - private static final String DESCRIPTION = "World Domination."; - private static final String ISSUE_53625_ASSAY = TestDataGenerator.randomDomainName("Issue53625", DomainUtils.DomainKind.Assay); - private static final String ISSUE_53625_PROJECT = "Issue53625Project"; - private static final String ISSUE_53616_ASSAY = "Issue53616Assay"; - private static final String ISSUE_53616_PROJECT = "Issue53616Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String ISSUE_53831_PROJECT = "Issue53831Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String SAMPLE_FIELD_TEST_ASSAY = "SampleFieldTestAssay"; - private static final String SAMPLE_FIELD_PROJECT_NAME = "Sample Field Test Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String MVTC_MULTI_FILE_IMPORT_ASSAY = TestDataGenerator.randomDomainName("MVTCMultiFileImportAssay", DomainUtils.DomainKind.Assay); - private static final String MVTC_MULTI_FILE_IMPORT_PROJECT = "MVTCMultiFileImportAssay" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String COL_ASSAY_ID_LABEL = "Assay ID"; - private static final List TEXT_MULTI_CHOICE_LIST = randomTextChoice(10); - private static final FieldInfo COL_MULTITEXTCHOICE = FieldInfo.random("Multi Choice", FieldDefinition.ColumnType.MultiValueTextChoice) - .customizeFieldDefinition(fd -> fd.setMultiChoiceValues(TEXT_MULTI_CHOICE_LIST)); - - - @Override - protected String getProjectName() - { - return TEST_ASSAY_PRJ_SECURITY; - } - - /** - * Cleanup entry point. - */ - @Override - protected void doCleanup(boolean afterTest) throws TestTimeoutException - { - //should also delete the groups - _containerHelper.deleteProject(getProjectName(), false); - _containerHelper.deleteProject(SAMPLE_FIELD_PROJECT_NAME, false); - _containerHelper.deleteProject(ISSUE_53616_PROJECT, false); - _containerHelper.deleteProject(ISSUE_53625_PROJECT, false); - _containerHelper.deleteProject(ISSUE_53831_PROJECT, false); - _containerHelper.deleteProject(MVTC_MULTI_FILE_IMPORT_PROJECT, false); - - _userHelper.deleteUsers(false, TEST_ASSAY_USR_PI1, TEST_ASSAY_USR_TECH1); - } - - // Issue 53831: Assay name max length check - @Test - public void testAssayNameMaxLength() throws Exception - { - _containerHelper.createProject(ISSUE_53831_PROJECT, "Assay"); - goToProjectHome(ISSUE_53831_PROJECT); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", "a" + "0123456789".repeat(15)); - List errors = assayDesignerPage.clickSaveExpectingErrors(); - checker().verifyEquals("Wrong number of errors", 1, errors.size()); - checker().verifyEquals("Wrong error message: " + errors.get(0), - "Value is too long for assay design name, a maximum length of 150 is allowed. The supplied value, 'a01234567890123456789012...78901234567890123456789', was 151 characters long.", - errors.get(0)); - assayDesignerPage.clickCancel(); - } - - @Test - public void testAssayMultiFileImportForMVTC() throws Exception - { - Assume.assumeTrue("Multi-choice text fields are only supported on PostgreSQL", WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL); - _containerHelper.createProject(MVTC_MULTI_FILE_IMPORT_PROJECT, "Assay"); - new GeneralAssayDesign(MVTC_MULTI_FILE_IMPORT_ASSAY) - .setRunFields(List.of(new FieldDefinition("runText", FieldDefinition.ColumnType.String)), true) - .setDataFields(List.of(COL_MULTITEXTCHOICE.getFieldDefinition()), false) - .createAssay(MVTC_MULTI_FILE_IMPORT_PROJECT, createDefaultConnection()); - - String firstFileName = "MVTCAssayImport.tsv"; - String secondFileName = "MVTCAssayImportSecond.tsv"; - List> fileDataFirstImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) - .limit(5) - .toList(); - List> fileDataSecondImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) - .limit(5) - .toList(); - - log("Import first and second runs with MVTC data from files"); - AssayImportPage assayImportPage = goToManageAssays() - .clickAssay(MVTC_MULTI_FILE_IMPORT_ASSAY) - .clickImportData(); - assayImportPage.clickNext(); - assayImportPage.setDataFile(writeMultiValueFileForAssayRun(firstFileName, fileDataFirstImport)); - - assayImportPage = assayImportPage.clickSaveAndImportAnother(); - assayImportPage.setDataFile(writeMultiValueFileForAssayRun(secondFileName, fileDataSecondImport)); - assayImportPage.clickSaveAndFinish(); - - AssayRunsPage assayRunsPage = new AssayRunsPage(getDriver()); - checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.getTable().getColumnDataAsText(COL_ASSAY_ID_LABEL)) - .as("expect both runs to appear in the runs list") - .containsExactlyInAnyOrder(firstFileName, secondFileName)); - - List expectedValues = Stream.concat(fileDataFirstImport.stream(), fileDataSecondImport.stream()) - .map(values -> TestArrayDataUtils.sortAndJoin(values, " ")) - .toList(); - checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.clickViewResults().getDataTable().getColumnDataAsText(COL_MULTITEXTCHOICE)) - .as("expect MVTC values to match imported data") - .containsExactlyInAnyOrderElementsOf(expectedValues)); - } - - private File writeMultiValueFileForAssayRun(String fileName, List> fileData) throws IOException - { - List> rows = Stream.concat( - Stream.of(List.of(COL_MULTITEXTCHOICE.getName())), - fileData.stream().map(row -> List.of(formatMultiValueText(row))) - ).toList(); - return TestDataUtils.writeRowsToFile(fileName, rows); - } - - // Issue 53616: Assay creation attempt after an error results in "Assay protocol already exists for this name." - @Test - public void testFailedCreation() throws Exception - { - _containerHelper.createProject(ISSUE_53616_PROJECT, "Assay"); - goToProjectHome(ISSUE_53616_PROJECT); - - log("Create test assay"); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", ISSUE_53616_ASSAY) - .setDescription(TEST_ASSAY_DESC); - - DomainFormPanel resultsPanel = assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset result fields - resultsPanel.addField("TooLongFieldName".repeat(20)); - - log("Save initial assay design with sample field set to 'All Samples'"); - List errors = assayDesignerPage.clickSaveExpectingErrors(); - assertEquals("Wrong number of errors", 1, errors.size()); - assertTrue("Wrong error message: " + errors.get(0), errors.get(0).startsWith("Name cannot exceed 200 characters, but was")); - - resultsPanel.removeAllFields(false); - resultsPanel.addField("ShortAndSweet"); - assayDesignerPage.clickFinish(); - - AssayListCommand command = new AssayListCommand(); - AssayListResponse response = command.execute(createDefaultConnection(), ISSUE_53616_PROJECT); - assertNotNull("Didn't find expected assay design", response.getDefinition(ISSUE_53616_ASSAY)); - } - - /** - * Performs the Assay security test - * This test creates a project with a folder hierarchy with multiple groups and users; - * defines an Assay at the project level; uploads run data as a labtech; publishes - * as a PI, and tests to make sure that security is properly enforced - */ - @Test - public void testAssaySecurity() throws Exception - { - log("Starting Assay security scenario tests"); - setupEnvironment(); - setupPipeline(getProjectName()); - SpecimenImporter importer = new SpecimenImporter(TestFileUtils.getTestTempDir(), StudyHelper.SPECIMEN_ARCHIVE_A, FileUtil.appendName(TestFileUtils.getTestTempDir(), "specimensSubDir"), TEST_ASSAY_FLDR_STUDY2, 1); - importer.importAndWaitForComplete(); - defineAssay(); - uploadRuns(TEST_ASSAY_FLDR_LAB1, TEST_ASSAY_USR_TECH1); - editResults(); - publishData(); - publishDataToDateBasedStudy(); - publishDataToVisitBasedStudy(); - editAssay(); - viewCrossFolderData(); - verifyStudyList(); - verifyRunDeletionRecallsDatasetRows(); - verifyWebdavTree(); - } - - @Test - public void testSampleFieldUpdate() - { - log("Starting sample field update test"); - _containerHelper.createProject(SAMPLE_FIELD_PROJECT_NAME, "Assay"); - - log("Create test assay"); - ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", SAMPLE_FIELD_TEST_ASSAY) - .setDescription(TEST_ASSAY_DESC); - - assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset batch fields - - DomainFormPanel resultsPanel = assayDesignerPage.goToResultsFields().removeAllFields(false); //remove preset result fields - - String sampleFieldName = "SampleField"; - resultsPanel.manuallyDefineFields(sampleFieldName) - .setType(FieldDefinition.ColumnType.Sample) - .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); - - log("Save initial assay design with sample field set to 'All Samples'"); - assayDesignerPage.clickFinish(); - - log("Verify save successful"); - assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); - AssayBeginPage assayPage = goToManageAssays(); - assertElementPresent(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - - log("Create new Sample Types to verify against"); - String targetTypeName = "Target Sample Type"; - SampleTypeDefinition targetDefinition = new SampleTypeDefinition(targetTypeName).setFields(new ArrayList<>()); - SampleTypeHelper ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); - ssHelper.createSampleType(targetDefinition, "Name\nS_1\nS_2\nS_3"); - - String otherTypeName = "Other Sample Type"; - SampleTypeDefinition otherDefinition = new SampleTypeDefinition(otherTypeName).setFields(new ArrayList<>()); - ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); - ssHelper.createSampleType(otherDefinition, "Name\nOS_1\nOS_2"); - - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN1, "SampleField\nOS_1"); - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); - - log("Edit assay design and change Sample field to point to created Sample Type"); - goToManageAssays(); - clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.expandFieldsPanel("Results") - .getField(sampleFieldName) - .setSampleType(targetTypeName); - designerPage.clickFinish(); - - log("Verify updates saved successfully"); - assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN2, "SampleField\nS_1"); - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable table = new DataRegionTable("Data", getDriver()); - List sampleFieldValues = table.getColumnDataAsText("SampleField"); - assertTrue("First sample should not resolve to sample type", sampleFieldValues.get(0).startsWith("<")); - assertEquals("Second sample should resolve to sample type", "S_1", sampleFieldValues.get(1)); - assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); - - log("GitHub Issue #688: verify sample lookup to createdBy"); - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("SampleField/CreatedBy"); - _customizeViewsHelper.applyCustomView(); - table = new DataRegionTable("Data", getDriver()); - List createdByValues = table.getColumnDataAsText("SampleField/CreatedBy"); - assertEquals("First sample should not have a createdBy since it doesn't resolve", " ", createdByValues.get(0)); - assertEquals("Second sample should have a createdBy since it resolves to a sample type", getCurrentUserName(), createdByValues.get(1)); - - log("Edit assay design and change Sample field to point back to 'All Samples'"); - goToManageAssays(); - clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.expandFieldsPanel("Results") - .getField(sampleFieldName) - .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); - designerPage.clickFinish(); - assertEquals("Error saving updated sample field", 0, checker().errorsSinceMark()); - - log("Verify updates saved successfully"); - importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN3, "SampleField\nS_2\nOS_2"); - assertEquals("Error importing data after assay sample field update", 0, checker().errorsSinceMark()); - - goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); - assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); - assertElementPresent("Sample lookup failed for: S_2", new Locator.LinkLocator("S_2"), 1); - assertElementPresent("Sample lookup failed for: OS_2", new Locator.LinkLocator("OS_2"), 1); - - log("GitHub Issue #688: verify sample lookup to createdBy"); - table = new DataRegionTable("Data", getDriver()); - for (int i = 0; i < table.getDataRowCount(); i++) - assertEquals("Row " + i + " should have current user as createdBy since they all resolve to samples", getCurrentUserName(), table.getDataAsText(i, "SampleField/CreatedBy")); - } - - private void importAssayData(String assayName, String runName, String runDataStr) - { - goToManageAssays(); - clickAndWait(Locator.linkWithText(assayName)); - clickButton("Import Data", "Run Data"); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, runName); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, runDataStr); - clickButton("Save and Finish"); - - } - - @LogMethod - private void verifyRunDeletionRecallsDatasetRows() - { - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - DataRegionTable assayRuns = new DataRegionTable("Runs", this); - assayRuns.checkCheckbox(0); - assayRuns.clickHeaderButtonAndWait("Delete"); - // Make sure that it shows that the data is part of study datasets - assertTextPresent(TEST_RUN3, "2 dataset(s)", TEST_ASSAY); - assertTextNotPresent("FirstRun"); - // Do the delete - clickButton("Confirm Delete"); - - // Be sure that we have a special audit record - clickAndWait(Locator.linkWithText("view link to study history")); - assertTextPresent("3 row(s) were recalled from a study to the assay: "); - - // Verify that the deleted run data is gone from the dataset - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); - clickAndWait(Locator.linkWithText("1 dataset")); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - assertTextPresent("AAA07XMC-04", TEST_RUN1); - assertTextNotPresent("BAQ00051-09", TEST_RUN3); - } - - //Issue 12203: Incorrect files are visible from pipeline directory - private void verifyWebdavTree() - { - beginAt("_webdav"); - _fileBrowserHelper.selectFileBrowserItem(getProjectName() + "/Studies/Study 1/"); - Locator.XPathLocator l = Locator.xpath("//span[text()='@pipeline']"); - assertElementPresent(l, 1); - } - - @LogMethod - private void editResults() throws IOException, CommandException - { - // Verify that the results aren't editable by default - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - waitAndClickAndWait(Locator.linkWithText("view results")); - DataRegionTable table = new DataRegionTable("Data", getDriver()); - assertEquals("No rows should be editable", 0, DataRegionTable.updateLinkLocator().findElements(table.getComponentElement()).size()); - assertElementNotPresent(Locator.button("Delete")); - - // Edit the design to make them editable - ReactAssayDesignerPage assayDesignerPage = _assayHelper.clickEditAssayDesign(true); - assayDesignerPage.setEditableResults(true); - assayDesignerPage.clickFinish(); - - // Try an edit - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - assertEquals("Incorrect number of results shown.", 10, table.getDataRowCount()); - doAndWaitForPageToLoad(() -> dataTable.updateLink(dataTable.getRowIndex("Specimen ID", "AAA07XK5-05")).click()); - setFormElement(Locator.name("quf_SpecimenID"), "EditedSpecimenID"); - setFormElement(Locator.name("quf_VisitID"), "601.5"); - setFormElement(Locator.name("quf_testAssayDataProp5"), "notAnumber"); - clickButton("Submit"); - assertTextPresent("Could not convert value: " + "notAnumber"); - setFormElement(Locator.name("quf_testAssayDataProp5"), "514801"); - clickButton("Submit"); - assertTextPresent("EditedSpecimenID", "601.5", "514801"); - - // Try a delete - dataTable.checkCheckbox(table.getRowIndex("Specimen ID", "EditedSpecimenID")); - doAndWaitForPageToLoad(() -> - { - dataTable.clickHeaderButton("Delete"); - assertAlert("Are you sure you want to delete the selected row?"); - }); - - // Verify that the edit was audited - AuditLogHelper auditLogHelper = new AuditLogHelper(this, () -> WebTestHelper.getRemoteApiConnection(false)); - auditLogHelper.checkAuditEventDiffCount(getProjectName(), AuditLogHelper.AuditEvent.QUERY_UPDATE_AUDIT_EVENT, List.of(0/*delete*/, 3/*edit*/)); - - goToSchemaBrowser(); - viewQueryData("auditLog", "ExperimentAuditEvent"); - assertTextPresent("1 data row has been edited in " + TEST_ASSAY + "."); - - } - - /** - * Generates the text that appears in the target study drop-down for a given study name - * @param studyName name of the target study - * @return formatted string of what appears in the target study drop-down - */ - private String getTargetStudyOptionText(String studyName) - { - //the format used in the drop down is: - // /// ( Study) - return "/" + getProjectName() + "/" + TEST_ASSAY_FLDR_STUDIES + "/" + - studyName + " (" + studyName + " Study)"; - } - - /** - * Uploads run data for the centrally defined Assay while impersonating a labtech-style user - * @param folder name of the folder into which we should upload - * @param asUser the user to impersonate before uploading - */ - @LogMethod - private void uploadRuns(String folder, String asUser) - { - log("Uploading runs into folder " + folder + " as user " + asUser); - navigateToFolder(getProjectName(), folder); - impersonate(asUser); - - clickAndWait(Locator.linkWithText("Assay List")); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - - //nav trail check - assertNavTrail("Assay List", TEST_ASSAY + " Batches"); - - clickButton("Import Data"); - assertTextPresent(TEST_ASSAY_SET_PROP_NAME + "3"); - - log("Batch properties"); - clickButton("Next"); - assertTextPresent(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + " is required and must be of type Number (Double)."); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), "Bad Test"); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), "Bad Test"); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), "Bad Test"); - clickButton("Next"); - assertTextPresent( - "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.", - "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.", - "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY)."); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]); - setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]); - - //ensure that the target study drop down contains Study 1 and Study 2 only and not Study 3 - //(labtech1 does not have read perms to Study 3) - waitForElement(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1))); - assertElementPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2))); - assertElementNotPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3))); - - //select Study2 as the target study (note that PI is not an Editor in this study so we can test for override case) - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); - - clickButton("Next"); - - log("Check properties set."); - assertTextPresent( - TEST_ASSAY_SET_PROPERTIES[1], - TEST_ASSAY_SET_PROPERTIES[2], - TEST_ASSAY_SET_PROPERTIES[3], - TEST_ASSAY_SET_PROPERTIES[0]); - - log("Run properties and data"); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC); - clickButton("Save and Finish"); - - assertTextPresent(TEST_ASSAY_RUN_PROP_NAME + "0 is required and must be of type Text (String)."); - assertTextPresent(PROTOCOL_DOC.getName()); - waitAndClick(Locator.linkWithText("remove")); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN1); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN1_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); - clickButton("Save and Finish"); - - Locator loc4 = Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"); - assertEquals("", getFormElement(loc4)); - assertTextPresent("Data file contained zero data rows"); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN1_DATA1); - clickButton("Save and Import Another Run"); - - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN2); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN2_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA1); - clickButton("Save and Finish"); - - assertTextPresent(PROTOCOL_DOC2.getName()); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA2); - clickButton("Save and Finish"); - - assertTextPresent("Could not convert value 'g' (String) for Double field 'VisitID'"); - assertTextPresent(PROTOCOL_DOC2.getName()); - assertEquals(TEST_RUN2, getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); - assertEquals(TEST_RUN2_COMMENTS, getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA3); - clickButton("Save and Import Another Run"); - - assertTextPresent("Missing value for required property: " + TEST_ASSAY_DATA_PROP_NAME + "6"); - click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA4); - clickButton("Save and Import Another Run"); - - assertEquals("", getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); - assertEquals("", getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); - setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN3); - setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN3_COMMENTS); - setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); - clickButton("Save and Finish"); - - assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); - setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN3_DATA1); - clickButton("Save and Finish"); - - // Verify the first run did not have a file, the second run had the attached file and the third run had a file - // with a unique name. - assertTextNotPresent(PROTOCOL_DOC.getName()); - assertTextPresent(PROTOCOL_DOC2.getName()); - assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); - - log("Check out the data for one of the runs"); - assertNoLabKeyErrors(); - assertTextPresent( - TEST_ASSAY + " Runs", - TEST_ASSAY_RUN_PROP1, - TEST_ASSAY_SET_PROPERTIES[0], - TEST_ASSAY_SET_PROPERTIES[3]); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - assertElementNotPresent(Locator.tagWithText("td", "7.0")); - // Make sure that our specimen IDs resolved correctly - assertTextPresent( - "AAA07XSF-02", - "999320885", - "301", - "AAA07XK5-05", - "999320812", - "601", - TEST_ASSAY_DATA_PROP_NAME + "4", - TEST_ASSAY_DATA_PROP_NAME + "5", - TEST_ASSAY_DATA_PROP_NAME + "6", - "2000-06-06", - "0.0", - "f", - ALIASED_DATA); - - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("SpecimenID/GlobalUniqueId"); - _customizeViewsHelper.addColumn("SpecimenID/Specimen/PrimaryType"); - _customizeViewsHelper.addColumn("SpecimenID/AssayMatch"); - _customizeViewsHelper.removeColumn("Run/testAssayRunProp1"); - _customizeViewsHelper.removeColumn("Run/Batch/testAssaySetProp2"); - _customizeViewsHelper.removeColumn("testAssayDataProp4"); - _customizeViewsHelper.applyCustomView(); - - assertTextPresent("Blood (Whole)", 4); - - Locator.XPathLocator trueLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'true']"); - int totalTrues = getElementCount(trueLocator); - assertEquals(4, totalTrues); - - DataRegionTable region = new DataRegionTable("Data", this); - region.setFilter("SpecimenID", "Starts With", "AssayTestControl"); - - // verify that there are no trues showing for the assay match column that were filtered out - totalTrues = getElementCount(trueLocator); - assertEquals(0, totalTrues); - - log("Check out the data for all of the runs"); - clickAndWait(Locator.linkWithText("view results")); - region.clearAllFilters("SpecimenID"); - assertElementPresent(Locator.tagWithText("td", "7.0")); - assertElementPresent(Locator.tagWithText("td", "18")); - - assertTextPresent("Blood (Whole)", 7); - - Locator.XPathLocator falseLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'false']"); - int totalFalses = getElementCount(falseLocator); - assertEquals(3, totalFalses); - - region.setFilter("SpecimenID", "Does Not Start With", "BAQ"); - - // verify the falses have been filtered out - totalFalses = getElementCount(falseLocator); - assertEquals(0, totalFalses); - - stopImpersonating(); - } - - /** - * Impersonates the PI user and publishes the data previous uploaded. - * This will also verify that the PI cannot publish to studies for which - * the PI does not have Editor permissions. - */ - @LogMethod - private void publishData() - { - log("Prepare visit map to check PTID counts in study navigator."); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); - _studyHelper.goToManageVisits().goToImportVisitMap(); - setFormElement(Locator.name("content"), - "\n" + - "\n" + - " \n" + - ""); - clickButton("Import"); - - log("Publishing the data as the PI"); - - //impersonate the PI - impersonate(TEST_ASSAY_USR_PI1); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", this); - table.checkAllOnPage(); - table.clickHeaderButtonAndWait("Link to Study"); - - //the target study selected before was Study2, but the PI is not an editor there - //so ensure that system has correctly caught this fact and now asks the PI to - //select a different study, and lists only those studies in which the PI is - //an editor - - //ensure warning - assertTextPresent("WARNING: You do not have permissions to link to one or more of the selected run's associated studies."); - - //ensure that Study2 and Study 3 are not available in the target study drop down - assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2) + "']")); - assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3) + "']")); - - //Study1 is the only one left, so it should be there and already be selected - assertElementPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + - getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1) + "']")); - - // Make sure the selected study is Study1 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY1 + " Study: Verify Results"); - - setFormElement(Locator.name("visitId"), "301.5"); - clickButton("Link to Study"); - - log("Verifying that the data was published"); - _customizeViewsHelper.openCustomizeViewPanel(); - _customizeViewsHelper.addColumn("QCState"); - _customizeViewsHelper.applyCustomView(); - assertTextPresent( - "Pending Review", - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "8", "1", "1", "1", "1", "1", "1", "2"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "8 / 9", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "2 / 3"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - doAndWaitForPageToLoad(() -> uncheckCheckbox(Locator.checkboxByNameAndValue("visitStatistic", "ParticipantCount"))); - row2 = new String[]{TEST_ASSAY, "9", "1", "1", "1", "1", "1", "1", "3"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - clickAndWait(Locator.linkWithText("9")); - - assertElementPresent(Locator.linkWithText("999320885"), 1); - assertElementPresent(Locator.linkWithText("999320885"), 1); - assertTextPresent( - "301.0", - "9.0", - "8.0", - TEST_RUN1_COMMENTS, - TEST_RUN2_COMMENTS, - TEST_RUN1, - TEST_RUN2, - "2000-06-06", - TEST_ASSAY_RUN_PROP1, - "18"); - - // test recall - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - waitAndClickAndWait(Locator.linkWithText("view link to study history")); - - // Set a filter so that we know we're recalling SecondRun - DataRegionTable region = new DataRegionTable("query", this); - region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); - doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); - - DataRegionTable linkStudy = new DataRegionTable("Dataset", this); - linkStudy.checkAll(); - doAndWaitForPageToLoad(() -> - { - linkStudy.clickHeaderButton("Recall Rows"); - acceptAlert(); - }); - assertTextPresent("row(s) were recalled from a study to the assay: " + TEST_ASSAY); - - // Set a filter so that we know we're looking at the link event for SecondRun again - region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); - - // verify audit entry was adjusted - doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); - assertTextPresent("All rows that were previously linked in this event have been recalled"); - - stopImpersonating(); - } - - /** - * Designed to test automatic timepoint generation when linking to a date based study. - * Most tests of timepoint matching are covered by separate junit tests; however, - * this will create 1 pre-existing timepoint, and when linking data this timepoint should be - * chosen for appropriate records. - */ - @LogMethod - private void publishDataToDateBasedStudy() - { - log("Prepare visit map to check PTID counts in study navigator."); - - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY3); - - clickAndWait(Locator.linkWithText("Manage")); - clickAndWait(Locator.linkWithText("Manage Timepoints")); - clickAndWait(Locator.linkWithText("Create New Timepoint")); - setFormElement(Locator.name("label"), "Preexisting Timepoint"); - setFormElement(Locator.name("sequenceNumMin"), "50"); - setFormElement(Locator.name("sequenceNumMax"), "89"); - selectOptionByText(Locator.name("typeCode"), "Screening"); - - clickButton("Save"); - assertElementPresent(Locator.tagWithAttribute("a", "data-original-title", "edit"), 1); - - //select the Lab1 folder and view all the data for the test assay - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", getDriver()); - table.checkAll(); - table.clickHeaderButtonAndWait("Link to Study"); - - checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); - - // Make sure the selected study is Study3 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY3 + " Study: Verify Results"); - - //populate initial set of values and verify the timepoint preview column - String[] dates = new String[]{"2000-02-02", "2000-03-03", "2000-04-04", "2000-05-05", "2000-06-06", "2001-01-01", "2000-01-01", "2000-02-02", "2000-03-03"}; - int idx = 1; - for (String d : dates) - { - setFormElement(Locator.xpath("(//input[@name='date'])[" + idx + "]"), d); - idx++; - } - - setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); - - DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); - linkStudy.clickHeaderButtonAndWait("Re-Validate"); - - //validate timepoints: - assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='AAA07XMC-02'] and following-sibling::td[text()='301.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='AAA07XMC-04'] and following-sibling::td[not(text())]]")); - assertElementPresent(Locator.xpath("//td[text()='Day 90 - 95' and following-sibling::td/a[text()='AAA07XSF-02'] and following-sibling::td[not(text())]]")); - - assertElementPresent(Locator.xpath("//td[text()='Day 120 - 127' and following-sibling::td/a[text()='AssayTestControl1'] and following-sibling::td[text()='5.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 152 - 159' and following-sibling::td/a[text()='AssayTestControl2'] and following-sibling::td[text()='6.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 0 - 7' and following-sibling::td/a[text()='BAQ00051-09'] and following-sibling::td[text()='7.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='BAQ00051-08'] and following-sibling::td[text()='8.0']]")); - assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='BAQ00051-11'] and following-sibling::td[text()='9.0']]")); - - linkStudy.clickHeaderButtonAndWait("Link to Study"); - - log("Verifying that the data was published"); - assertTextPresent( - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "9", "1", "2", "2", "1", "1", "1"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "9 / 9", "1 / 1", "2 / 2", "2 / 2", "1 / 1", "1 / 1", "1 / 1"}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - log("Test that correct timepoints were created"); - - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Manage Study")); - clickAndWait(Locator.linkWithText("Manage Timepoints")); - assertTextPresent( - "Day 0 - 7", - "Day 32 - 39", - "Day 90 - 95", - "Day 120 - 127", - "Day 152 - 159"); - } - - - /** - * Designed to test automatic timepoint generation when linking to a date based study. - * Most tests of timepoint matching are covered by separate junit tests; however, - * this will create 1 pre-existing timepoint, and when linking data this timepoint should be - * chosen for appropriate records. - */ - @LogMethod - private void publishDataToVisitBasedStudy() - { - log("Prepare visit map to check PTID counts in study navigator."); - - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); - - _studyHelper.goToManageVisits().goToImportVisitMap(); - setFormElement(Locator.name("content"), - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - "" - ); - clickButton("Import"); - - //select the Lab1 folder and view all the data for the test assay - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText("view results")); - - //select all the data rows and click publish - DataRegionTable table = new DataRegionTable("Data", getDriver()); - table.checkAll(); - table.clickHeaderButtonAndWait("Link to Study"); - - checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); - - // Make sure the selected study is Study2 - selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); - - clickButton("Next"); - assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY2 + " Study: Verify Results"); - - //populate initial set of values and verify the timepoint preview column - String[] visits = new String[]{"302", "33", "4", "70"}; - int idx = 1; - for (String v : visits) - { - setFormElement(Locator.xpath("(//input[@name='visitId'])[" + idx + "]"), v); - idx++; - } - - setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); - setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); - - DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); - linkStudy.clickHeaderButtonAndWait("Re-Validate"); - - //validate timepoints: - assertElementPresent(Locator.xpath("//td[text()='Test Visit3' and following-sibling::td/a[text()='AAA07XMC-02']]")); - assertElementPresent(Locator.xpath("//td[text()='33.0' and following-sibling::td/a[text()='AAA07XMC-04']]")); - assertElementPresent(Locator.xpath("//td[text()='4.0' and following-sibling::td/a[text()='AAA07XSF-02']]")); - - assertElementPresent(Locator.xpath("//td[text()='Test Visit2' and following-sibling::td/a[text()='AssayTestControl1']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='AssayTestControl2']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-09']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-08']]")); - assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-11']]")); - - linkStudy.clickHeaderButtonAndWait("Link to Study"); - - log("Verifying that the data was published"); - assertTextPresent( - TEST_RUN1_COMMENTS, - "2000-01-01"); - clickTab("Overview"); - clickAndWait(Locator.linkWithText("Study Navigator")); - - log("Test participant counts and row counts in study overview"); - String[] row2 = new String[]{TEST_ASSAY, "9", " ", " ", " ", "1", " ", " ", "1", " ", " ", "4", " ", " ", " ", " ", "1", "1", " ", " ", " ", "1", " ", " ", " ", " ", " "}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event - clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); - row2 = new String[]{TEST_ASSAY, "9 / 9", " ", " ", " ", "1 / 1", " ", " ", "1 / 1", " ", " ", "4 / 4", " ", " ", " ", " ", "1 / 1", "1 / 1", " ", " ", " ", "1 / 1", " ", " ", " ", " ", " "}; - assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); - - log("Test that correct timepoints were created"); - - clickTab("Overview"); - _studyHelper.goToManageVisits(); - assertTextPresent( - "Test Visit1", - "6.0 - 13.0", - "Test Visit2", - "50.0 - 70.0", - "Test Visit3", - "302.0 - 303.0"); - } - - /** - * Tests editing of an existing assay definition - */ - @LogMethod - private void editAssay() - { - log("Testing edit and delete and assay definition"); - clickProject(getProjectName()); - waitAndClickAndWait(Locator.linkWithText(TEST_ASSAY)); - - // change a field name and label and remove a field - ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); - DomainFormPanel domainFormPanel = designerPage.expandFieldsPanel("Results"); - domainFormPanel.getField(5).setName(TEST_ASSAY_DATA_PROP_NAME + "edit"); - domainFormPanel.getField(5).setLabel(TEST_ASSAY_DATA_PROP_NAME + "edit"); - domainFormPanel.removeField(domainFormPanel.getField(4).getName(), true); - designerPage.clickFinish(); - - //ensure that label has changed in run data in Lab 1 folder - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - assertTextPresent(TEST_ASSAY_DATA_PROP_NAME + "edit"); - assertTextNotPresent(TEST_ASSAY_DATA_PROP_NAME + 4); - - AuditLogTest.verifyAuditEvent(this, AuditLogTest.ASSAY_AUDIT_EVENT, AuditLogTest.COMMENT_COLUMN, "were linked to a study from the assay: " + TEST_ASSAY, 5); - } - - @LogMethod - private void viewCrossFolderData() - { - log("Testing cross-folder data"); - - clickProject(getProjectName()); - - portalHelper.addWebPart("Assay Runs"); - selectOptionByText(Locator.name("viewProtocolId"), "General: " + TEST_ASSAY); - // assay runs has a details page that needs to be submitted - clickButton("Submit", defaultWaitForPage); - - // Set the container filter to include subfolders - DataRegionTable assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - assayRuns.setContainerFilter(DataRegionTable.ContainerFilterType.CURRENT_AND_SUBFOLDERS); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - log("Save the customized view to include subfolders"); - assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - CustomizeView customizeViewsHelper = assayRuns.getCustomizeView(); - customizeViewsHelper.openCustomizeViewPanel(); - customizeViewsHelper.saveCustomView(""); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - log("Testing select all data and view"); - assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); - assayRuns.checkAllOnPage(); - clickButton("Show Results", defaultWaitForPage); - verifySpecimensPresent(3, 2, 3); - - log("Testing clicking on a run"); - clickProject(getProjectName()); - clickAndWait(Locator.linkWithText(TEST_RUN1)); - verifySpecimensPresent(3, 2, 0); - - clickAndWait(Locator.linkWithText("view results")); - DataRegionTable region = new DataRegionTable("Data", this); - region.clearAllFilters("SpecimenID"); - verifySpecimensPresent(3, 2, 3); - - log("Testing assay-study linkage"); - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); - portalHelper.addWebPart("Datasets"); - clickAndWait(Locator.linkWithText(TEST_ASSAY)); - clickButton("View Source Assay", defaultWaitForPage); - - assertTextPresent(TEST_RUN1, TEST_RUN2); - - clickAndWait(Locator.linkWithText(TEST_RUN1)); - verifySpecimensPresent(3, 2, 0); - - clickAndWait(Locator.linkWithText("view results")); - region = new DataRegionTable("Data", this); - region.clearAllFilters("SpecimenID"); - verifySpecimensPresent(3, 2, 3); - - // Verify that the correct linked to study column is present - assertTextPresent("Linked to Study 1 Study"); - - log("Testing link to study availability"); - clickProject(getProjectName()); - clickAndWait(Locator.linkWithText(TEST_RUN3)); - - region = new DataRegionTable("Data", this); - region.checkAll(); - region.clickHeaderButtonAndWait("Link to Study"); - clickButton("Next"); - - verifySpecimensPresent(0, 0, 3); - - clickButton("Cancel"); - } - - @LogMethod - private void verifyStudyList() - { - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); - portalHelper.addWebPart("Study List"); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY2 + " Study")); - assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY3 + " Study")); - portalHelper.clickWebpartMenuItem("Studies", "Customize"); - - //verify grid view - selectOptionByText(Locator.name("displayType"), "Grid"); - clickButton("Submit"); - assertElementNotPresent(Locator.linkWithText("edit")); - - //edit study properties - clickAndWait(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); - click(Locator.tagWithAttribute("span", "title", "Edit")); - waitForElement(Locator.name("Investigator"), WAIT_FOR_JAVASCRIPT); - setFormElement(Locator.name("Investigator"), INVESTIGATOR); - setFormElement(Locator.name("Grant"), GRANT); - setFormElement(Locator.name("Description"), DESCRIPTION); - clickButton("Submit"); - - //verify study properties (grid view) - navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); - DataRegionTable table = new DataRegionTable("qwpStudies", this); - assertEquals("Studies not sorted correctly.", TEST_ASSAY_FLDR_STUDY1 + " Study", table.getDataAsText(0, "Label")); - assertEquals("Failed to set study investigator.", INVESTIGATOR, table.getDataAsText(0, "Investigator")); - assertEquals("Failed to set study grant.", GRANT, table.getDataAsText(0, "Grant")); - assertEquals("Failed to set study description.", DESCRIPTION, table.getDataAsText(0, "Description")); - - //verify study properties (details view) - portalHelper.clickWebpartMenuItem("Studies", "Customize"); - selectOptionByText(Locator.name("displayType"), "Details"); - clickButton("Submit"); - assertTextPresent(INVESTIGATOR, DESCRIPTION); - assertTextNotPresent(GRANT, TEST_ASSAY_FLDR_STUDY1 + " Study tracks data"); //Old description - } - - private void verifySpecimensPresent(int aaa07Count, int controlCount, int baq00051Count) - { - // need to double the count, once for the label and once for the param in the link url - assertTextPresent("AAA07", aaa07Count * 2); - assertTextPresent("AssayTestControl", controlCount * 2); - assertTextPresent("BAQ00051", baq00051Count * 2); - } - - @Test // Issue 53625 - public void testAssayLookupValidatorConversion() - { - _containerHelper.createProject(ISSUE_53625_PROJECT, "Assay"); - goToProjectHome(ISSUE_53625_PROJECT); - - log("Create a list with an int key and a string value"); - String lookToListName = TestDataGenerator.randomDomainName("lookToList", DomainUtils.DomainKind.IntList); - String keyName = TestDataGenerator.randomFieldName("key", null, DomainUtils.DomainKind.IntList); - FieldInfo valueField = FieldInfo.random("value", FieldDefinition.ColumnType.String, DomainUtils.DomainKind.IntList); - _listHelper.createList(ISSUE_53625_PROJECT, lookToListName, keyName, valueField.getFieldDefinition()); - _listHelper.bulkImportData(TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(valueField.getName(), "One"), - Map.of(valueField.getName(), "Two"), - Map.of(valueField.getName(), "123"), - // GitHub Issue #443: value is the primary key for another row - Map.of(valueField.getName(), "5"), // pk = 4 - Map.of(valueField.getName(), "6") // pk = 5 - ), List.of(valueField.getName()), true)); - - log("Create an assay with a results lookup field to the list, with lookup validator set"); - goToProjectHome(ISSUE_53625_PROJECT); - FieldInfo lookupField = new FieldInfo("lookup", new FieldDefinition.IntLookup(null, "lists", lookToListName)); - ReactAssayDesignerPage designerPage = _assayHelper.createAssayDesign("General", ISSUE_53625_ASSAY); - designerPage.goToBatchFields() - .removeAllFields(false); - designerPage.goToResultsFields() - .removeAllFields(false) - .manuallyDefineFields(lookupField.getFieldDefinition().setLookupValidatorEnabled(true)); - designerPage.clickFinish(); - - log("Verify importing an assay run with valid and invalid values for the lookup field"); - verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithLookupValidator", true); - - log("Turn off lookup field validator and test the imports again"); - designerPage = _assayHelper.clickEditAssayDesign(); - designerPage.goToResultsFields() - .getField(lookupField.getName()) - .setLookupValidatorEnabled(false); - designerPage.clickFinish(); - verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithoutLookupValidator", false); - - log("GitHub Issue #443: Verify that importing a value that is also a primary key maps to the titleColumn value"); - verifyAssayImportForPKValueThatIsTitleColumn(ISSUE_53625_ASSAY, lookupField, "RunWithPKandTitleColumn"); - } - - private void verifyAssayImportForPKValueThatIsTitleColumn(String assayName, FieldInfo lookupField, String runName) - { - String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "4"), // pk 4, value 5 - Map.of(lookupField.getName(), "5"), // pk 4, value 5 - Map.of(lookupField.getName(), "6")), // pk 5, value 6 - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); - checker().fatal().verifyEquals("Lookup values not as expected.", List.of("5", "5", "6"), dataTable.getColumnDataAsText(lookupField.getLabel())); - } - - private void verifyAssayImportForLookupValidator(String assayName, FieldInfo lookupField, String runName, boolean validatorOn) - { - String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "One"), // valid - Map.of(lookupField.getName(), "99")), // invalid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - assertTextPresent("Could not translate value: 99"); - if (validatorOn) assertTextPresent("Value '99' was not present in lookup target"); - else assertTextNotPresent("Value '99' was not present in lookup target"); - clickButton("Cancel"); - - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "Three"), // invalid - Map.of(lookupField.getName(), "2")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - assertTextPresent("Failed to convert"); - assertTextPresent("Could not translate value: Three"); - clickButton("Cancel"); - - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "One"), // valid - Map.of(lookupField.getName(), "2"), // valid - Map.of(lookupField.getName(), "123")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); - checker().verifyEquals("Lookup values not as expected.", List.of("One", "Two", "123"), dataTable.getColumnDataAsText(lookupField.getLabel())); - - // test with just the numeric value since that was causing issues during manual testing - runName = runName + "NumericOnly"; - runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( - Map.of(lookupField.getName(), "123")), // valid - List.of(lookupField.getName()), true - ); - importAssayData(assayName, runName, runDataStr); - clickAndWait(Locator.linkWithText(runName)); - dataTable = new DataRegionTable("Data", getDriver()); - checker().verifyEquals("Incorrect number of results shown.", 1, dataTable.getDataRowCount()); - checker().verifyEquals("Lookup values not as expected.", List.of("123"), dataTable.getColumnDataAsText(lookupField.getLabel())); - } - - @Override - protected BrowserType bestBrowser() - { - return BrowserType.CHROME; - } -} +/* + * Copyright (c) 2016-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.study; + +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.assay.AssayListCommand; +import org.labkey.remoteapi.assay.AssayListResponse; +import org.labkey.test.Locator; +import org.labkey.test.TestFileUtils; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.WebTestHelper; +import org.labkey.test.categories.Assays; +import org.labkey.test.categories.Daily; +import org.labkey.test.components.CustomizeView; +import org.labkey.test.components.assay.AssayConstants; +import org.labkey.test.components.domain.DomainFieldRow; +import org.labkey.test.components.domain.DomainFormPanel; +import org.labkey.test.pages.ReactAssayDesignerPage; +import org.labkey.test.pages.assay.AssayBeginPage; +import org.labkey.test.pages.assay.AssayImportPage; +import org.labkey.test.pages.assay.AssayRunsPage; +import org.labkey.test.params.FieldDefinition; +import org.labkey.test.params.FieldInfo; +import org.labkey.test.params.assay.GeneralAssayDesign; +import org.labkey.test.params.experiment.SampleTypeDefinition; +import org.labkey.test.tests.AbstractAssayTest; +import org.labkey.test.tests.AuditLogTest; +import org.labkey.test.util.AuditLogHelper; +import org.labkey.test.util.DataRegionTable; +import org.labkey.test.util.DomainUtils; +import org.labkey.test.util.LogMethod; +import org.labkey.test.util.OptionalFeatureHelper; +import org.labkey.test.util.SampleTypeHelper; +import org.labkey.test.util.StudyHelper; +import org.labkey.test.util.TestDataGenerator; +import org.labkey.test.util.data.TestArrayDataUtils; +import org.labkey.test.util.data.TestDataUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.labkey.test.util.TestDataGenerator.randomTextChoice; +import static org.labkey.test.util.TestDataGenerator.shuffleSelect; +import static org.labkey.test.util.data.TestArrayDataUtils.formatMultiValueText; + +@Category({Daily.class, Assays.class}) +public class AssayTest extends AbstractAssayTest +{ + private static final String INVESTIGATOR = "Dr. No"; + private static final String GRANT = "SPECTRE"; + private static final String DESCRIPTION = "World Domination."; + private static final String ISSUE_53625_ASSAY = TestDataGenerator.randomDomainName("Issue53625", DomainUtils.DomainKind.Assay); + private static final String ISSUE_53625_PROJECT = "Issue53625Project"; + private static final String ISSUE_53616_ASSAY = "Issue53616Assay"; + private static final String ISSUE_53616_PROJECT = "Issue53616Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String ISSUE_53831_PROJECT = "Issue53831Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String SAMPLE_FIELD_TEST_ASSAY = "SampleFieldTestAssay"; + private static final String SAMPLE_FIELD_PROJECT_NAME = "Sample Field Test Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String MVTC_MULTI_FILE_IMPORT_ASSAY = TestDataGenerator.randomDomainName("MVTCMultiFileImportAssay", DomainUtils.DomainKind.Assay); + private static final String MVTC_MULTI_FILE_IMPORT_PROJECT = "MVTCMultiFileImportAssay" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String COL_ASSAY_ID_LABEL = "Assay ID"; + private static final List TEXT_MULTI_CHOICE_LIST = randomTextChoice(10); + private static final FieldInfo COL_MULTITEXTCHOICE = FieldInfo.random("Multi Choice", FieldDefinition.ColumnType.MultiValueTextChoice) + .customizeFieldDefinition(fd -> fd.setMultiChoiceValues(TEXT_MULTI_CHOICE_LIST)); + + + @Override + protected String getProjectName() + { + return TEST_ASSAY_PRJ_SECURITY; + } + + /** + * Cleanup entry point. + */ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + //should also delete the groups + _containerHelper.deleteProject(getProjectName(), false); + _containerHelper.deleteProject(SAMPLE_FIELD_PROJECT_NAME, false); + _containerHelper.deleteProject(ISSUE_53616_PROJECT, false); + _containerHelper.deleteProject(ISSUE_53625_PROJECT, false); + _containerHelper.deleteProject(ISSUE_53831_PROJECT, false); + _containerHelper.deleteProject(MVTC_MULTI_FILE_IMPORT_PROJECT, false); + + _userHelper.deleteUsers(false, TEST_ASSAY_USR_PI1, TEST_ASSAY_USR_TECH1); + } + + // Issue 53831: Assay name max length check + @Test + public void testAssayNameMaxLength() throws Exception + { + _containerHelper.createProject(ISSUE_53831_PROJECT, "Assay"); + goToProjectHome(ISSUE_53831_PROJECT); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", "a" + "0123456789".repeat(15)); + List errors = assayDesignerPage.clickSaveExpectingErrors(); + checker().verifyEquals("Wrong number of errors", 1, errors.size()); + checker().verifyEquals("Wrong error message: " + errors.get(0), + "Value is too long for assay design name, a maximum length of 150 is allowed. The supplied value, 'a01234567890123456789012...78901234567890123456789', was 151 characters long.", + errors.get(0)); + assayDesignerPage.clickCancel(); + } + + @Test + public void testAssayMultiFileImportForMVTC() throws Exception + { + Assume.assumeTrue("Multi-choice text fields are only supported on PostgreSQL", WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL); + _containerHelper.createProject(MVTC_MULTI_FILE_IMPORT_PROJECT, "Assay"); + new GeneralAssayDesign(MVTC_MULTI_FILE_IMPORT_ASSAY) + .setRunFields(List.of(new FieldDefinition("runText", FieldDefinition.ColumnType.String)), true) + .setDataFields(List.of(COL_MULTITEXTCHOICE.getFieldDefinition()), false) + .createAssay(MVTC_MULTI_FILE_IMPORT_PROJECT, createDefaultConnection()); + + String firstFileName = "MVTCAssayImport.tsv"; + String secondFileName = "MVTCAssayImportSecond.tsv"; + List> fileDataFirstImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) + .limit(5) + .toList(); + List> fileDataSecondImport = Stream.generate(() -> shuffleSelect(TEXT_MULTI_CHOICE_LIST)) + .limit(5) + .toList(); + + log("Import first and second runs with MVTC data from files"); + AssayImportPage assayImportPage = goToManageAssays() + .clickAssay(MVTC_MULTI_FILE_IMPORT_ASSAY) + .clickImportData(); + assayImportPage.clickNext(); + assayImportPage.setDataFile(writeMultiValueFileForAssayRun(firstFileName, fileDataFirstImport)); + + assayImportPage = assayImportPage.clickSaveAndImportAnother(); + assayImportPage.setDataFile(writeMultiValueFileForAssayRun(secondFileName, fileDataSecondImport)); + assayImportPage.clickSaveAndFinish(); + + AssayRunsPage assayRunsPage = new AssayRunsPage(getDriver()); + checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.getTable().getColumnDataAsText(COL_ASSAY_ID_LABEL)) + .as("expect both runs to appear in the runs list") + .containsExactlyInAnyOrder(firstFileName, secondFileName)); + + List expectedValues = Stream.concat(fileDataFirstImport.stream(), fileDataSecondImport.stream()) + .map(values -> TestArrayDataUtils.sortAndJoin(values, " ")) + .toList(); + checker().wrapAssertion(() -> Assertions.assertThat(assayRunsPage.clickViewResults().getDataTable().getColumnDataAsText(COL_MULTITEXTCHOICE)) + .as("expect MVTC values to match imported data") + .containsExactlyInAnyOrderElementsOf(expectedValues)); + } + + private File writeMultiValueFileForAssayRun(String fileName, List> fileData) throws IOException + { + List> rows = Stream.concat( + Stream.of(List.of(COL_MULTITEXTCHOICE.getName())), + fileData.stream().map(row -> List.of(formatMultiValueText(row))) + ).toList(); + return TestDataUtils.writeRowsToFile(fileName, rows); + } + + // Issue 53616: Assay creation attempt after an error results in "Assay protocol already exists for this name." + @Test + public void testFailedCreation() throws Exception + { + _containerHelper.createProject(ISSUE_53616_PROJECT, "Assay"); + goToProjectHome(ISSUE_53616_PROJECT); + + log("Create test assay"); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", ISSUE_53616_ASSAY) + .setDescription(TEST_ASSAY_DESC); + + DomainFormPanel resultsPanel = assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset result fields + resultsPanel.addField("TooLongFieldName".repeat(20)); + + log("Save initial assay design with sample field set to 'All Samples'"); + List errors = assayDesignerPage.clickSaveExpectingErrors(); + assertEquals("Wrong number of errors", 1, errors.size()); + assertTrue("Wrong error message: " + errors.get(0), errors.get(0).startsWith("Name cannot exceed 200 characters, but was")); + + resultsPanel.removeAllFields(false); + resultsPanel.addField("ShortAndSweet"); + assayDesignerPage.clickFinish(); + + AssayListCommand command = new AssayListCommand(); + AssayListResponse response = command.execute(createDefaultConnection(), ISSUE_53616_PROJECT); + assertNotNull("Didn't find expected assay design", response.getDefinition(ISSUE_53616_ASSAY)); + } + + /** + * Performs the Assay security test + * This test creates a project with a folder hierarchy with multiple groups and users; + * defines an Assay at the project level; uploads run data as a labtech; publishes + * as a PI, and tests to make sure that security is properly enforced + */ + @Test + public void testAssaySecurity() throws Exception + { + log("Starting Assay security scenario tests"); + setupEnvironment(); + setupPipeline(getProjectName()); + SpecimenImporter importer = new SpecimenImporter(TestFileUtils.getTestTempDir(), StudyHelper.SPECIMEN_ARCHIVE_A, FileUtil.appendName(TestFileUtils.getTestTempDir(), "specimensSubDir"), TEST_ASSAY_FLDR_STUDY2, 1); + importer.importAndWaitForComplete(); + defineAssay(); + uploadRuns(TEST_ASSAY_FLDR_LAB1, TEST_ASSAY_USR_TECH1); + editResults(); + publishData(); + publishDataToDateBasedStudy(); + publishDataToVisitBasedStudy(); + editAssay(); + viewCrossFolderData(); + verifyStudyList(); + verifyRunDeletionRecallsDatasetRows(); + verifyWebdavTree(); + } + + @Test + public void testSampleFieldUpdate() + { + log("Starting sample field update test"); + _containerHelper.createProject(SAMPLE_FIELD_PROJECT_NAME, "Assay"); + + log("Create test assay"); + ReactAssayDesignerPage assayDesignerPage = _assayHelper.createAssayDesign("General", SAMPLE_FIELD_TEST_ASSAY) + .setDescription(TEST_ASSAY_DESC); + + assayDesignerPage.goToBatchFields().removeAllFields(false); //remove preset batch fields + + DomainFormPanel resultsPanel = assayDesignerPage.goToResultsFields().removeAllFields(false); //remove preset result fields + + String sampleFieldName = "SampleField"; + resultsPanel.manuallyDefineFields(sampleFieldName) + .setType(FieldDefinition.ColumnType.Sample) + .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); + + log("Save initial assay design with sample field set to 'All Samples'"); + assayDesignerPage.clickFinish(); + + log("Verify save successful"); + assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); + AssayBeginPage assayPage = goToManageAssays(); + assertElementPresent(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + + log("Create new Sample Types to verify against"); + String targetTypeName = "Target Sample Type"; + SampleTypeDefinition targetDefinition = new SampleTypeDefinition(targetTypeName).setFields(new ArrayList<>()); + SampleTypeHelper ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); + ssHelper.createSampleType(targetDefinition, "Name\nS_1\nS_2\nS_3"); + + String otherTypeName = "Other Sample Type"; + SampleTypeDefinition otherDefinition = new SampleTypeDefinition(otherTypeName).setFields(new ArrayList<>()); + ssHelper = SampleTypeHelper.beginAtSampleTypesList(this, getCurrentContainerPath()); + ssHelper.createSampleType(otherDefinition, "Name\nOS_1\nOS_2"); + + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN1, "SampleField\nOS_1"); + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); + + log("Edit assay design and change Sample field to point to created Sample Type"); + goToManageAssays(); + clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.expandFieldsPanel("Results") + .getField(sampleFieldName) + .setSampleType(targetTypeName); + designerPage.clickFinish(); + + log("Verify updates saved successfully"); + assertEquals("Error saving initial assay", 0, checker().errorsSinceMark()); + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN2, "SampleField\nS_1"); + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable table = new DataRegionTable("Data", getDriver()); + List sampleFieldValues = table.getColumnDataAsText("SampleField"); + assertTrue("First sample should not resolve to sample type", sampleFieldValues.get(0).startsWith("<")); + assertEquals("Second sample should resolve to sample type", "S_1", sampleFieldValues.get(1)); + assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); + + log("GitHub Issue #688: verify sample lookup to createdBy"); + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("SampleField/CreatedBy"); + _customizeViewsHelper.applyCustomView(); + table = new DataRegionTable("Data", getDriver()); + List createdByValues = table.getColumnDataAsText("SampleField/CreatedBy"); + assertEquals("First sample should not have a createdBy since it doesn't resolve", " ", createdByValues.get(0)); + assertEquals("Second sample should have a createdBy since it resolves to a sample type", getCurrentUserName(), createdByValues.get(1)); + + log("Edit assay design and change Sample field to point back to 'All Samples'"); + goToManageAssays(); + clickAndWait(Locator.LinkLocator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.expandFieldsPanel("Results") + .getField(sampleFieldName) + .setSampleType(DomainFieldRow.ALL_SAMPLES_OPTION_TEXT); + designerPage.clickFinish(); + assertEquals("Error saving updated sample field", 0, checker().errorsSinceMark()); + + log("Verify updates saved successfully"); + importAssayData(SAMPLE_FIELD_TEST_ASSAY, TEST_RUN3, "SampleField\nS_2\nOS_2"); + assertEquals("Error importing data after assay sample field update", 0, checker().errorsSinceMark()); + + goToManageAssays().clickAndWait(Locator.linkWithText(SAMPLE_FIELD_TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + assertElementPresent("Sample lookup failed for: OS_1", new Locator.LinkLocator("OS_1"), 1); + assertElementPresent("Sample lookup failed for: S_1", new Locator.LinkLocator("S_1"), 1); + assertElementPresent("Sample lookup failed for: S_2", new Locator.LinkLocator("S_2"), 1); + assertElementPresent("Sample lookup failed for: OS_2", new Locator.LinkLocator("OS_2"), 1); + + log("GitHub Issue #688: verify sample lookup to createdBy"); + table = new DataRegionTable("Data", getDriver()); + for (int i = 0; i < table.getDataRowCount(); i++) + assertEquals("Row " + i + " should have current user as createdBy since they all resolve to samples", getCurrentUserName(), table.getDataAsText(i, "SampleField/CreatedBy")); + } + + private void importAssayData(String assayName, String runName, String runDataStr) + { + goToManageAssays(); + clickAndWait(Locator.linkWithText(assayName)); + clickButton("Import Data", "Run Data"); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, runName); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, runDataStr); + clickButton("Save and Finish"); + + } + + @LogMethod + private void verifyRunDeletionRecallsDatasetRows() + { + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + DataRegionTable assayRuns = new DataRegionTable("Runs", this); + assayRuns.checkCheckbox(0); + assayRuns.clickHeaderButtonAndWait("Delete"); + // Make sure that it shows that the data is part of study datasets + assertTextPresent(TEST_RUN3, "2 dataset(s)", TEST_ASSAY); + assertTextNotPresent("FirstRun"); + // Do the delete + clickButton("Confirm Delete"); + + // Be sure that we have a special audit record + clickAndWait(Locator.linkWithText("view link to study history")); + assertTextPresent("3 row(s) were recalled from a study to the assay: "); + + // Verify that the deleted run data is gone from the dataset + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); + clickAndWait(Locator.linkWithText("1 dataset")); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + assertTextPresent("AAA07XMC-04", TEST_RUN1); + assertTextNotPresent("BAQ00051-09", TEST_RUN3); + } + + //Issue 12203: Incorrect files are visible from pipeline directory + private void verifyWebdavTree() + { + beginAt("_webdav"); + _fileBrowserHelper.selectFileBrowserItem(getProjectName() + "/Studies/Study 1/"); + Locator.XPathLocator l = Locator.xpath("//span[text()='@pipeline']"); + assertElementPresent(l, 1); + } + + @LogMethod + private void editResults() throws IOException, CommandException + { + // Verify that the results aren't editable by default + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + waitAndClickAndWait(Locator.linkWithText("view results")); + DataRegionTable table = new DataRegionTable("Data", getDriver()); + assertEquals("No rows should be editable", 0, DataRegionTable.updateLinkLocator().findElements(table.getComponentElement()).size()); + assertElementNotPresent(Locator.button("Delete")); + + // Edit the design to make them editable + ReactAssayDesignerPage assayDesignerPage = _assayHelper.clickEditAssayDesign(true); + assayDesignerPage.setEditableResults(true); + assayDesignerPage.clickFinish(); + + // Try an edit + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + assertEquals("Incorrect number of results shown.", 10, table.getDataRowCount()); + doAndWaitForPageToLoad(() -> dataTable.updateLink(dataTable.getRowIndex("Specimen ID", "AAA07XK5-05")).click()); + setFormElement(Locator.name("quf_SpecimenID"), "EditedSpecimenID"); + setFormElement(Locator.name("quf_VisitID"), "601.5"); + setFormElement(Locator.name("quf_testAssayDataProp5"), "notAnumber"); + clickButton("Submit"); + assertTextPresent("Could not convert value: " + "notAnumber"); + setFormElement(Locator.name("quf_testAssayDataProp5"), "514801"); + clickButton("Submit"); + assertTextPresent("EditedSpecimenID", "601.5", "514801"); + + // Try a delete + dataTable.checkCheckbox(table.getRowIndex("Specimen ID", "EditedSpecimenID")); + doAndWaitForPageToLoad(() -> + { + dataTable.clickHeaderButton("Delete"); + assertAlert("Are you sure you want to delete the selected row?"); + }); + + // Verify that the edit was audited + AuditLogHelper auditLogHelper = new AuditLogHelper(this, () -> WebTestHelper.getRemoteApiConnection(false)); + auditLogHelper.checkAuditEventDiffCount(getProjectName(), AuditLogHelper.AuditEvent.QUERY_UPDATE_AUDIT_EVENT, List.of(0/*delete*/, 3/*edit*/)); + + goToSchemaBrowser(); + viewQueryData("auditLog", "ExperimentAuditEvent"); + assertTextPresent("1 data row has been edited in " + TEST_ASSAY + "."); + + } + + /** + * Generates the text that appears in the target study drop-down for a given study name + * @param studyName name of the target study + * @return formatted string of what appears in the target study drop-down + */ + private String getTargetStudyOptionText(String studyName) + { + //the format used in the drop down is: + // /// ( Study) + return "/" + getProjectName() + "/" + TEST_ASSAY_FLDR_STUDIES + "/" + + studyName + " (" + studyName + " Study)"; + } + + /** + * Uploads run data for the centrally defined Assay while impersonating a labtech-style user + * @param folder name of the folder into which we should upload + * @param asUser the user to impersonate before uploading + */ + @LogMethod + private void uploadRuns(String folder, String asUser) + { + log("Uploading runs into folder " + folder + " as user " + asUser); + navigateToFolder(getProjectName(), folder); + impersonate(asUser); + + clickAndWait(Locator.linkWithText("Assay List")); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + + //nav trail check + assertNavTrail("Assay List", TEST_ASSAY + " Batches"); + + clickButton("Import Data"); + assertTextPresent(TEST_ASSAY_SET_PROP_NAME + "3"); + + log("Batch properties"); + clickButton("Next"); + assertTextPresent(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + " is required and must be of type Number (Double)."); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), "Bad Test"); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), "Bad Test"); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), "Bad Test"); + clickButton("Next"); + assertTextPresent( + "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.", + "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.", + "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY)."); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]); + setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]); + + //ensure that the target study drop down contains Study 1 and Study 2 only and not Study 3 + //(labtech1 does not have read perms to Study 3) + waitForElement(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1))); + assertElementPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2))); + assertElementNotPresent(Locator.xpath("//option").withText(getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3))); + + //select Study2 as the target study (note that PI is not an Editor in this study so we can test for override case) + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); + + clickButton("Next"); + + log("Check properties set."); + assertTextPresent( + TEST_ASSAY_SET_PROPERTIES[1], + TEST_ASSAY_SET_PROPERTIES[2], + TEST_ASSAY_SET_PROPERTIES[3], + TEST_ASSAY_SET_PROPERTIES[0]); + + log("Run properties and data"); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC); + clickButton("Save and Finish"); + + assertTextPresent(TEST_ASSAY_RUN_PROP_NAME + "0 is required and must be of type Text (String)."); + assertTextPresent(PROTOCOL_DOC.getName()); + waitAndClick(Locator.linkWithText("remove")); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN1); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN1_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); + clickButton("Save and Finish"); + + Locator loc4 = Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"); + assertEquals("", getFormElement(loc4)); + assertTextPresent("Data file contained zero data rows"); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN1_DATA1); + clickButton("Save and Import Another Run"); + + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN2); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN2_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "0"), TEST_ASSAY_RUN_PROP1); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA1); + clickButton("Save and Finish"); + + assertTextPresent(PROTOCOL_DOC2.getName()); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA2); + clickButton("Save and Finish"); + + assertTextPresent("Could not convert value 'g' (String) for Double field 'VisitID'"); + assertTextPresent(PROTOCOL_DOC2.getName()); + assertEquals(TEST_RUN2, getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); + assertEquals(TEST_RUN2_COMMENTS, getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA3); + clickButton("Save and Import Another Run"); + + assertTextPresent("Missing value for required property: " + TEST_ASSAY_DATA_PROP_NAME + "6"); + click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN2_DATA4); + clickButton("Save and Import Another Run"); + + assertEquals("", getFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR)); + assertEquals("", getFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR)); + setFormElement(AssayConstants.ASSAY_NAME_FIELD_LOCATOR, TEST_RUN3); + setFormElement(AssayConstants.COMMENTS_FIELD_LOCATOR, TEST_RUN3_COMMENTS); + setFormElement(Locator.name(TEST_ASSAY_RUN_PROP_NAME + "5"), PROTOCOL_DOC2); + clickButton("Save and Finish"); + + assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); + setFormElement(AssayConstants.TEXT_AREA_DATA_COLLECTOR_LOCATOR, TEST_RUN3_DATA1); + clickButton("Save and Finish"); + + // Verify the first run did not have a file, the second run had the attached file and the third run had a file + // with a unique name. + assertTextNotPresent(PROTOCOL_DOC.getName()); + assertTextPresent(PROTOCOL_DOC2.getName()); + assertTextPresent(PROTOCOL_DOC2.getName().substring(0, PROTOCOL_DOC2.getName().lastIndexOf(".")) + "-1"); + + log("Check out the data for one of the runs"); + assertNoLabKeyErrors(); + assertTextPresent( + TEST_ASSAY + " Runs", + TEST_ASSAY_RUN_PROP1, + TEST_ASSAY_SET_PROPERTIES[0], + TEST_ASSAY_SET_PROPERTIES[3]); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + assertElementNotPresent(Locator.tagWithText("td", "7.0")); + // Make sure that our specimen IDs resolved correctly + assertTextPresent( + "AAA07XSF-02", + "999320885", + "301", + "AAA07XK5-05", + "999320812", + "601", + TEST_ASSAY_DATA_PROP_NAME + "4", + TEST_ASSAY_DATA_PROP_NAME + "5", + TEST_ASSAY_DATA_PROP_NAME + "6", + "2000-06-06", + "0.0", + "f", + ALIASED_DATA); + + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("SpecimenID/GlobalUniqueId"); + _customizeViewsHelper.addColumn("SpecimenID/Specimen/PrimaryType"); + _customizeViewsHelper.addColumn("SpecimenID/AssayMatch"); + _customizeViewsHelper.removeColumn("Run/testAssayRunProp1"); + _customizeViewsHelper.removeColumn("Run/Batch/testAssaySetProp2"); + _customizeViewsHelper.removeColumn("testAssayDataProp4"); + _customizeViewsHelper.applyCustomView(); + + assertTextPresent("Blood (Whole)", 4); + + Locator.XPathLocator trueLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'true']"); + int totalTrues = getElementCount(trueLocator); + assertEquals(4, totalTrues); + + DataRegionTable region = new DataRegionTable("Data", this); + region.setFilter("SpecimenID", "Starts With", "AssayTestControl"); + + // verify that there are no trues showing for the assay match column that were filtered out + totalTrues = getElementCount(trueLocator); + assertEquals(0, totalTrues); + + log("Check out the data for all of the runs"); + clickAndWait(Locator.linkWithText("view results")); + region.clearAllFilters("SpecimenID"); + assertElementPresent(Locator.tagWithText("td", "7.0")); + assertElementPresent(Locator.tagWithText("td", "18")); + + assertTextPresent("Blood (Whole)", 7); + + Locator.XPathLocator falseLocator = Locator.xpath("//table[contains(@class, 'labkey-data-region')]//td[text() = 'false']"); + int totalFalses = getElementCount(falseLocator); + assertEquals(3, totalFalses); + + region.setFilter("SpecimenID", "Does Not Start With", "BAQ"); + + // verify the falses have been filtered out + totalFalses = getElementCount(falseLocator); + assertEquals(0, totalFalses); + + stopImpersonating(); + } + + /** + * Impersonates the PI user and publishes the data previous uploaded. + * This will also verify that the PI cannot publish to studies for which + * the PI does not have Editor permissions. + */ + @LogMethod + private void publishData() + { + log("Prepare visit map to check PTID counts in study navigator."); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); + _studyHelper.goToManageVisits().goToImportVisitMap(); + setFormElement(Locator.name("content"), + "\n" + + "\n" + + " \n" + + ""); + clickButton("Import"); + + log("Publishing the data as the PI"); + + //impersonate the PI + impersonate(TEST_ASSAY_USR_PI1); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", this); + table.checkAllOnPage(); + table.clickHeaderButtonAndWait("Link to Study"); + + //the target study selected before was Study2, but the PI is not an editor there + //so ensure that system has correctly caught this fact and now asks the PI to + //select a different study, and lists only those studies in which the PI is + //an editor + + //ensure warning + assertTextPresent("WARNING: You do not have permissions to link to one or more of the selected run's associated studies."); + + //ensure that Study2 and Study 3 are not available in the target study drop down + assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2) + "']")); + assertElementNotPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3) + "']")); + + //Study1 is the only one left, so it should be there and already be selected + assertElementPresent(Locator.xpath("//select[@name='TargetStudy']/option[.='" + + getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1) + "']")); + + // Make sure the selected study is Study1 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY1)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY1 + " Study: Verify Results"); + + setFormElement(Locator.name("visitId"), "301.5"); + clickButton("Link to Study"); + + log("Verifying that the data was published"); + _customizeViewsHelper.openCustomizeViewPanel(); + _customizeViewsHelper.addColumn("QCState"); + _customizeViewsHelper.applyCustomView(); + assertTextPresent( + "Pending Review", + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "8", "1", "1", "1", "1", "1", "1", "2"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "8 / 9", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "1 / 1", "2 / 3"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + doAndWaitForPageToLoad(() -> uncheckCheckbox(Locator.checkboxByNameAndValue("visitStatistic", "ParticipantCount"))); + row2 = new String[]{TEST_ASSAY, "9", "1", "1", "1", "1", "1", "1", "3"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + clickAndWait(Locator.linkWithText("9")); + + assertElementPresent(Locator.linkWithText("999320885"), 1); + assertElementPresent(Locator.linkWithText("999320885"), 1); + assertTextPresent( + "301.0", + "9.0", + "8.0", + TEST_RUN1_COMMENTS, + TEST_RUN2_COMMENTS, + TEST_RUN1, + TEST_RUN2, + "2000-06-06", + TEST_ASSAY_RUN_PROP1, + "18"); + + // test recall + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + waitAndClickAndWait(Locator.linkWithText("view link to study history")); + + // Set a filter so that we know we're recalling SecondRun + DataRegionTable region = new DataRegionTable("query", this); + region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); + doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); + + DataRegionTable linkStudy = new DataRegionTable("Dataset", this); + linkStudy.checkAll(); + doAndWaitForPageToLoad(() -> + { + linkStudy.clickHeaderButton("Recall Rows"); + acceptAlert(); + }); + assertTextPresent("row(s) were recalled from a study to the assay: " + TEST_ASSAY); + + // Set a filter so that we know we're looking at the link event for SecondRun again + region.setFilter("Comment", "Starts With", "3 row(s) were linked to a study from the assay"); + + // verify audit entry was adjusted + doAndWaitForPageToLoad(() -> region.detailsLink(region.getRowIndex("Assay/Protocol", TEST_ASSAY)).click()); + assertTextPresent("All rows that were previously linked in this event have been recalled"); + + stopImpersonating(); + } + + /** + * Designed to test automatic timepoint generation when linking to a date based study. + * Most tests of timepoint matching are covered by separate junit tests; however, + * this will create 1 pre-existing timepoint, and when linking data this timepoint should be + * chosen for appropriate records. + */ + @LogMethod + private void publishDataToDateBasedStudy() + { + log("Prepare visit map to check PTID counts in study navigator."); + + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY3); + + clickAndWait(Locator.linkWithText("Manage")); + clickAndWait(Locator.linkWithText("Manage Timepoints")); + clickAndWait(Locator.linkWithText("Create New Timepoint")); + setFormElement(Locator.name("label"), "Preexisting Timepoint"); + setFormElement(Locator.name("sequenceNumMin"), "50"); + setFormElement(Locator.name("sequenceNumMax"), "89"); + selectOptionByText(Locator.name("typeCode"), "Screening"); + + clickButton("Save"); + assertElementPresent(Locator.tagWithAttribute("a", "data-original-title", "edit"), 1); + + //select the Lab1 folder and view all the data for the test assay + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", getDriver()); + table.checkAll(); + table.clickHeaderButtonAndWait("Link to Study"); + + checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); + + // Make sure the selected study is Study3 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY3)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY3 + " Study: Verify Results"); + + //populate initial set of values and verify the timepoint preview column + String[] dates = new String[]{"2000-02-02", "2000-03-03", "2000-04-04", "2000-05-05", "2000-06-06", "2001-01-01", "2000-01-01", "2000-02-02", "2000-03-03"}; + int idx = 1; + for (String d : dates) + { + setFormElement(Locator.xpath("(//input[@name='date'])[" + idx + "]"), d); + idx++; + } + + setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); + + DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); + linkStudy.clickHeaderButtonAndWait("Re-Validate"); + + //validate timepoints: + assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='AAA07XMC-02'] and following-sibling::td[text()='301.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='AAA07XMC-04'] and following-sibling::td[not(text())]]")); + assertElementPresent(Locator.xpath("//td[text()='Day 90 - 95' and following-sibling::td/a[text()='AAA07XSF-02'] and following-sibling::td[not(text())]]")); + + assertElementPresent(Locator.xpath("//td[text()='Day 120 - 127' and following-sibling::td/a[text()='AssayTestControl1'] and following-sibling::td[text()='5.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 152 - 159' and following-sibling::td/a[text()='AssayTestControl2'] and following-sibling::td[text()='6.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 0 - 7' and following-sibling::td/a[text()='BAQ00051-09'] and following-sibling::td[text()='7.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Day 32 - 39' and following-sibling::td/a[text()='BAQ00051-08'] and following-sibling::td[text()='8.0']]")); + assertElementPresent(Locator.xpath("//td[text()='Preexisting Timepoint' and following-sibling::td/a[text()='BAQ00051-11'] and following-sibling::td[text()='9.0']]")); + + linkStudy.clickHeaderButtonAndWait("Link to Study"); + + log("Verifying that the data was published"); + assertTextPresent( + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "9", "1", "2", "2", "1", "1", "1"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "9 / 9", "1 / 1", "2 / 2", "2 / 2", "1 / 1", "1 / 1", "1 / 1"}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + log("Test that correct timepoints were created"); + + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Manage Study")); + clickAndWait(Locator.linkWithText("Manage Timepoints")); + assertTextPresent( + "Day 0 - 7", + "Day 32 - 39", + "Day 90 - 95", + "Day 120 - 127", + "Day 152 - 159"); + } + + + /** + * Designed to test automatic timepoint generation when linking to a date based study. + * Most tests of timepoint matching are covered by separate junit tests; however, + * this will create 1 pre-existing timepoint, and when linking data this timepoint should be + * chosen for appropriate records. + */ + @LogMethod + private void publishDataToVisitBasedStudy() + { + log("Prepare visit map to check PTID counts in study navigator."); + + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY2); + + _studyHelper.goToManageVisits().goToImportVisitMap(); + setFormElement(Locator.name("content"), + "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + "" + ); + clickButton("Import"); + + //select the Lab1 folder and view all the data for the test assay + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText("view results")); + + //select all the data rows and click publish + DataRegionTable table = new DataRegionTable("Data", getDriver()); + table.checkAll(); + table.clickHeaderButtonAndWait("Link to Study"); + + checkCheckbox(Locator.xpath("//input[@id='chooseStudy']")); + + // Make sure the selected study is Study2 + selectOptionByText(AssayConstants.TARGET_STUDY_FIELD_LOCATOR, getTargetStudyOptionText(TEST_ASSAY_FLDR_STUDY2)); + + clickButton("Next"); + assertTextPresent("Link to " + TEST_ASSAY_FLDR_STUDY2 + " Study: Verify Results"); + + //populate initial set of values and verify the timepoint preview column + String[] visits = new String[]{"302", "33", "4", "70"}; + int idx = 1; + for (String v : visits) + { + setFormElement(Locator.xpath("(//input[@name='visitId'])[" + idx + "]"), v); + idx++; + } + + setFormElement(Locator.xpath("(//input[@name='participantId'])[1]"), "new1"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[2]"), "new2"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[3]"), "new3"); + setFormElement(Locator.xpath("(//input[@name='participantId'])[4]"), "new4"); + + DataRegionTable linkStudy = new DataRegionTable("Data", getDriver()); + linkStudy.clickHeaderButtonAndWait("Re-Validate"); + + //validate timepoints: + assertElementPresent(Locator.xpath("//td[text()='Test Visit3' and following-sibling::td/a[text()='AAA07XMC-02']]")); + assertElementPresent(Locator.xpath("//td[text()='33.0' and following-sibling::td/a[text()='AAA07XMC-04']]")); + assertElementPresent(Locator.xpath("//td[text()='4.0' and following-sibling::td/a[text()='AAA07XSF-02']]")); + + assertElementPresent(Locator.xpath("//td[text()='Test Visit2' and following-sibling::td/a[text()='AssayTestControl1']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='AssayTestControl2']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-09']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-08']]")); + assertElementPresent(Locator.xpath("//td[text()='Test Visit1' and following-sibling::td/a[text()='BAQ00051-11']]")); + + linkStudy.clickHeaderButtonAndWait("Link to Study"); + + log("Verifying that the data was published"); + assertTextPresent( + TEST_RUN1_COMMENTS, + "2000-01-01"); + clickTab("Overview"); + clickAndWait(Locator.linkWithText("Study Navigator")); + + log("Test participant counts and row counts in study overview"); + String[] row2 = new String[]{TEST_ASSAY, "9", " ", " ", " ", "1", " ", " ", "1", " ", " ", "4", " ", " ", " ", " ", "1", "1", " ", " ", " ", "1", " ", " ", " ", " ", " "}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + // Manually click the checkbox -- normal checkCheckbox() method doesn't seem to work for checkbox that reloads using onchange event + clickAndWait(Locator.checkboxByNameAndValue("visitStatistic", "RowCount")); + row2 = new String[]{TEST_ASSAY, "9 / 9", " ", " ", " ", "1 / 1", " ", " ", "1 / 1", " ", " ", "4 / 4", " ", " ", " ", " ", "1 / 1", "1 / 1", " ", " ", " ", "1 / 1", " ", " ", " ", " ", " "}; + assertTableRowsEqual("studyOverview", 1, new String[][]{row2}); + + log("Test that correct timepoints were created"); + + clickTab("Overview"); + _studyHelper.goToManageVisits(); + assertTextPresent( + "Test Visit1", + "6.0 - 13.0", + "Test Visit2", + "50.0 - 70.0", + "Test Visit3", + "302.0 - 303.0"); + } + + /** + * Tests editing of an existing assay definition + */ + @LogMethod + private void editAssay() + { + log("Testing edit and delete and assay definition"); + clickProject(getProjectName()); + waitAndClickAndWait(Locator.linkWithText(TEST_ASSAY)); + + // change a field name and label and remove a field + ReactAssayDesignerPage designerPage = _assayHelper.clickEditAssayDesign(); + DomainFormPanel domainFormPanel = designerPage.expandFieldsPanel("Results"); + domainFormPanel.getField(5).setName(TEST_ASSAY_DATA_PROP_NAME + "edit"); + domainFormPanel.getField(5).setLabel(TEST_ASSAY_DATA_PROP_NAME + "edit"); + domainFormPanel.removeField(domainFormPanel.getField(4).getName(), true); + designerPage.clickFinish(); + + //ensure that label has changed in run data in Lab 1 folder + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_LAB1); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + assertTextPresent(TEST_ASSAY_DATA_PROP_NAME + "edit"); + assertTextNotPresent(TEST_ASSAY_DATA_PROP_NAME + 4); + + AuditLogTest.verifyAuditEvent(this, AuditLogTest.ASSAY_AUDIT_EVENT, AuditLogTest.COMMENT_COLUMN, "were linked to a study from the assay: " + TEST_ASSAY, 5); + } + + @LogMethod + private void viewCrossFolderData() + { + log("Testing cross-folder data"); + + clickProject(getProjectName()); + + portalHelper.addWebPart("Assay Runs"); + selectOptionByText(Locator.name("viewProtocolId"), "General: " + TEST_ASSAY); + // assay runs has a details page that needs to be submitted + clickButton("Submit", defaultWaitForPage); + + // Set the container filter to include subfolders + DataRegionTable assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + assayRuns.setContainerFilter(DataRegionTable.ContainerFilterType.CURRENT_AND_SUBFOLDERS); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + log("Save the customized view to include subfolders"); + assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + CustomizeView customizeViewsHelper = assayRuns.getCustomizeView(); + customizeViewsHelper.openCustomizeViewPanel(); + customizeViewsHelper.saveCustomView(""); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + log("Testing select all data and view"); + assayRuns = DataRegionTable.findDataRegionWithinWebpart(this, TEST_ASSAY + " Runs"); + assayRuns.checkAllOnPage(); + clickButton("Show Results", defaultWaitForPage); + verifySpecimensPresent(3, 2, 3); + + log("Testing clicking on a run"); + clickProject(getProjectName()); + clickAndWait(Locator.linkWithText(TEST_RUN1)); + verifySpecimensPresent(3, 2, 0); + + clickAndWait(Locator.linkWithText("view results")); + DataRegionTable region = new DataRegionTable("Data", this); + region.clearAllFilters("SpecimenID"); + verifySpecimensPresent(3, 2, 3); + + log("Testing assay-study linkage"); + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDY1); + portalHelper.addWebPart("Datasets"); + clickAndWait(Locator.linkWithText(TEST_ASSAY)); + clickButton("View Source Assay", defaultWaitForPage); + + assertTextPresent(TEST_RUN1, TEST_RUN2); + + clickAndWait(Locator.linkWithText(TEST_RUN1)); + verifySpecimensPresent(3, 2, 0); + + clickAndWait(Locator.linkWithText("view results")); + region = new DataRegionTable("Data", this); + region.clearAllFilters("SpecimenID"); + verifySpecimensPresent(3, 2, 3); + + // Verify that the correct linked to study column is present + assertTextPresent("Linked to Study 1 Study"); + + log("Testing link to study availability"); + clickProject(getProjectName()); + clickAndWait(Locator.linkWithText(TEST_RUN3)); + + region = new DataRegionTable("Data", this); + region.checkAll(); + region.clickHeaderButtonAndWait("Link to Study"); + clickButton("Next"); + + verifySpecimensPresent(0, 0, 3); + + clickButton("Cancel"); + } + + @LogMethod + private void verifyStudyList() + { + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); + portalHelper.addWebPart("Study List"); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY2 + " Study")); + assertElementPresent(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY3 + " Study")); + portalHelper.clickWebpartMenuItem("Studies", "Customize"); + + //verify grid view + selectOptionByText(Locator.name("displayType"), "Grid"); + clickButton("Submit"); + assertElementNotPresent(Locator.linkWithText("edit")); + + //edit study properties + clickAndWait(Locator.linkWithText(TEST_ASSAY_FLDR_STUDY1 + " Study")); + click(Locator.tagWithAttribute("span", "title", "Edit")); + waitForElement(Locator.name("Investigator"), WAIT_FOR_JAVASCRIPT); + setFormElement(Locator.name("Investigator"), INVESTIGATOR); + setFormElement(Locator.name("Grant"), GRANT); + setFormElement(Locator.name("Description"), DESCRIPTION); + clickButton("Submit"); + + //verify study properties (grid view) + navigateToFolder(getProjectName(), TEST_ASSAY_FLDR_STUDIES); + DataRegionTable table = new DataRegionTable("qwpStudies", this); + assertEquals("Studies not sorted correctly.", TEST_ASSAY_FLDR_STUDY1 + " Study", table.getDataAsText(0, "Label")); + assertEquals("Failed to set study investigator.", INVESTIGATOR, table.getDataAsText(0, "Investigator")); + assertEquals("Failed to set study grant.", GRANT, table.getDataAsText(0, "Grant")); + assertEquals("Failed to set study description.", DESCRIPTION, table.getDataAsText(0, "Description")); + + //verify study properties (details view) + portalHelper.clickWebpartMenuItem("Studies", "Customize"); + selectOptionByText(Locator.name("displayType"), "Details"); + clickButton("Submit"); + assertTextPresent(INVESTIGATOR, DESCRIPTION); + assertTextNotPresent(GRANT, TEST_ASSAY_FLDR_STUDY1 + " Study tracks data"); //Old description + } + + private void verifySpecimensPresent(int aaa07Count, int controlCount, int baq00051Count) + { + // need to double the count, once for the label and once for the param in the link url + assertTextPresent("AAA07", aaa07Count * 2); + assertTextPresent("AssayTestControl", controlCount * 2); + assertTextPresent("BAQ00051", baq00051Count * 2); + } + + @Test // Issue 53625 + public void testAssayLookupValidatorConversion() + { + _containerHelper.createProject(ISSUE_53625_PROJECT, "Assay"); + goToProjectHome(ISSUE_53625_PROJECT); + + log("Create a list with an int key and a string value"); + String lookToListName = TestDataGenerator.randomDomainName("lookToList", DomainUtils.DomainKind.IntList); + String keyName = TestDataGenerator.randomFieldName("key", null, DomainUtils.DomainKind.IntList); + FieldInfo valueField = FieldInfo.random("value", FieldDefinition.ColumnType.String, DomainUtils.DomainKind.IntList); + _listHelper.createList(ISSUE_53625_PROJECT, lookToListName, keyName, valueField.getFieldDefinition()); + _listHelper.bulkImportData(TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(valueField.getName(), "One"), + Map.of(valueField.getName(), "Two"), + Map.of(valueField.getName(), "123"), + // GitHub Issue #443: value is the primary key for another row + Map.of(valueField.getName(), "5"), // pk = 4 + Map.of(valueField.getName(), "6") // pk = 5 + ), List.of(valueField.getName()), true)); + + log("Create an assay with a results lookup field to the list, with lookup validator set"); + goToProjectHome(ISSUE_53625_PROJECT); + FieldInfo lookupField = new FieldInfo("lookup", new FieldDefinition.IntLookup(null, "lists", lookToListName)); + ReactAssayDesignerPage designerPage = _assayHelper.createAssayDesign("General", ISSUE_53625_ASSAY); + designerPage.goToBatchFields() + .removeAllFields(false); + designerPage.goToResultsFields() + .removeAllFields(false) + .manuallyDefineFields(lookupField.getFieldDefinition().setLookupValidatorEnabled(true)); + designerPage.clickFinish(); + + log("Verify importing an assay run with valid and invalid values for the lookup field"); + verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithLookupValidator", true); + + log("Turn off lookup field validator and test the imports again"); + designerPage = _assayHelper.clickEditAssayDesign(); + designerPage.goToResultsFields() + .getField(lookupField.getName()) + .setLookupValidatorEnabled(false); + designerPage.clickFinish(); + verifyAssayImportForLookupValidator(ISSUE_53625_ASSAY, lookupField, "RunWithoutLookupValidator", false); + + log("GitHub Issue #443: Verify that importing a value that is also a primary key maps to the titleColumn value"); + verifyAssayImportForPKValueThatIsTitleColumn(ISSUE_53625_ASSAY, lookupField, "RunWithPKandTitleColumn"); + } + + private void verifyAssayImportForPKValueThatIsTitleColumn(String assayName, FieldInfo lookupField, String runName) + { + String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "4"), // pk 4, value 5 + Map.of(lookupField.getName(), "5"), // pk 4, value 5 + Map.of(lookupField.getName(), "6")), // pk 5, value 6 + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); + checker().fatal().verifyEquals("Lookup values not as expected.", List.of("5", "5", "6"), dataTable.getColumnDataAsText(lookupField.getLabel())); + } + + private void verifyAssayImportForLookupValidator(String assayName, FieldInfo lookupField, String runName, boolean validatorOn) + { + String runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "One"), // valid + Map.of(lookupField.getName(), "99")), // invalid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + assertTextPresent("Could not translate value: 99"); + if (validatorOn) assertTextPresent("Value '99' was not present in lookup target"); + else assertTextNotPresent("Value '99' was not present in lookup target"); + clickButton("Cancel"); + + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "Three"), // invalid + Map.of(lookupField.getName(), "2")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + assertTextPresent("Failed to convert"); + assertTextPresent("Could not translate value: Three"); + clickButton("Cancel"); + + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "One"), // valid + Map.of(lookupField.getName(), "2"), // valid + Map.of(lookupField.getName(), "123")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + DataRegionTable dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 3, dataTable.getDataRowCount()); + checker().verifyEquals("Lookup values not as expected.", List.of("One", "Two", "123"), dataTable.getColumnDataAsText(lookupField.getLabel())); + + // test with just the numeric value since that was causing issues during manual testing + runName = runName + "NumericOnly"; + runDataStr = TestDataUtils.tsvStringFromRowMaps(List.of( + Map.of(lookupField.getName(), "123")), // valid + List.of(lookupField.getName()), true + ); + importAssayData(assayName, runName, runDataStr); + clickAndWait(Locator.linkWithText(runName)); + dataTable = new DataRegionTable("Data", getDriver()); + checker().verifyEquals("Incorrect number of results shown.", 1, dataTable.getDataRowCount()); + checker().verifyEquals("Lookup values not as expected.", List.of("123"), dataTable.getColumnDataAsText(lookupField.getLabel())); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } +} From fd43326f1a7d2c0ef693a3afbe67384db4349162 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 26 Apr 2026 17:00:46 -0700 Subject: [PATCH 3/3] crlf --- api/src/org/labkey/api/settings/AppProps.java | 518 +++++++++--------- 1 file changed, 259 insertions(+), 259 deletions(-) diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index 7ebe2bd9266..c6112cc0f27 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -1,259 +1,259 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.settings; - -import jakarta.servlet.http.HttpServletRequest; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.SupportedDatabase; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.Path; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.view.ActionURL; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Stores basic site-wide configuration. - * @see org.labkey.api.settings.WriteableAppProps - */ -public interface AppProps -{ - AppProps _instance = new AppPropsImpl(); - - String SCOPE_SITE_SETTINGS = "SiteSettings"; - - // Used for all optional features; "experimental" for historical reasons. - String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; - String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. - String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; - String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; - String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; - String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; - String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; - String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; - - String UNKNOWN_VERSION = "Unknown Release Version"; - - static AppProps getInstance() - { - return _instance; - } - - static WriteableAppProps getWriteableInstance() - { - return new WriteableAppProps(ContainerManager.getRoot()); - } - - String getServerSessionGUID(); - - boolean isMailRecorderEnabled(); - - boolean isOptionalFeatureEnabled(String feature); - - boolean isDevMode(); - - @Nullable - String getEnlistmentId(); - - boolean isCachingAllowed(); - - boolean isRecompileJspEnabled(); - - /** - * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode - * without the risk of loading unwanted resources from a source tree that may not match the deployed server. - * - * WARNING: Setting this flag will interfere with the population of module beans, resulting in a - * mismatch between deployed modules and their properties on the server. - * - * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false - * - * @see org.labkey.api.module.DefaultModule#setSourcePath(String) - * @see org.labkey.api.module.DefaultModule#setBuildPath(String) - * @see DefaultModule#computeResourceDirectory() - */ - boolean isIgnoreModuleSource(); - - void setProjectRoot(String projectRoot); - - /** - * @return the root of the main source tree - */ - @Nullable - String getProjectRoot(); - - /** - * @return directory under which all containers will automatically have their own subdirectory for storing files - */ - @Nullable - File getFileSystemRoot(); - - @NotNull - UsageReportingLevel getUsageReportingLevel(); - - /** - * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". - * Or "Unknown Release Version". - */ - @NotNull - String getReleaseVersion(); - - /** - * Convenience method for getting the core schema version, returning 0.0 instead of null - */ - double getSchemaVersion(); - - String getContextPath(); - - Path getParsedContextPath(); - - int getServerPort(); - - String getScheme(); - - String getServerName(); - - /** - * Save the current request URL if the base server URL property is not set - */ - void ensureBaseServerUrl(HttpServletRequest request); - - void setContextPath(String contextPath); - - boolean isSetBaseServerUrl(); - - String getBaseServerUrl(); - - String getHomePageUrl(); - - ActionURL getHomePageActionURL(); - - String getSiteWelcomePageUrlString(); - - int getLookAndFeelRevision(); - - String getDefaultLsidAuthority(); - - String getPipelineToolsDirectory(); - - boolean isSSLRequired(); - - boolean isUserRequestedAdminOnlyMode(); - - String getAdminOnlyMessage(); - - boolean isShowRibbonMessage(); - - @Nullable String getRibbonMessage(); - - int getSSLPort(); - - int getMemoryUsageDumpInterval(); - - /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ - int getReadOnlyHttpRequestTimeout(); - - int getMaxBLOBSize(); - - boolean isExt3Required(); - - boolean isExt3APIRequired(); - - ExceptionReportingLevel getExceptionReportingLevel(); - - /** - * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full - * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. - * - * @return if navigation access is open - */ - boolean isNavigationAccessOpen(); - - boolean isSelfReportExceptions(); - - String getServerGUID(); - - String getBLASTServerBaseURL(); - - /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ - String getWebappConfigurationFilename(); - - /** - * Email address of the primary site or application administrator, set on the site settings page. Useful in error - * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., - * only impersonating troubleshooters). - * - * @return Email address of the primary site or application administrator - */ - @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); - - boolean isAllowApiKeys(); - - int getApiKeyExpirationSeconds(); - - boolean isAllowSessionKeys(); - - // configurable http security settings - - /** - * @return "SAMEORIGIN" or "DENY" or "ALLOW" - */ - String getXFrameOption(); - - String getStaticFilesPrefix(); - - boolean isWebfilesRootEnabled(); - - boolean isFileUploadDisabled(); - - boolean isInvalidFilenameUploadBlocked(); - - boolean isInvalidFilenameBlocked(); - - /** @return whether the server should include its name and version as a header in HTTP responses */ - boolean isIncludeServerHttpHeader(); - - /** - * @return List of configured external redirect hosts - */ - @NotNull - List getExternalRedirectHosts(); - - /** - * @return List of configured external resource hosts - */ - @Deprecated // Left for upgrade code only - @NotNull - List getExternalSourceHosts(); - - Map getStashedStartupProperties(); - - @NotNull String getDistributionName(); - - @NotNull String getDistributionFilename(); - - @NotNull Set getDistributionSupportedDatabases(); - - @NotNull List getAllowedExtensions(); - - @NotNull String getAllowedExternalResourceHosts(); -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.settings; + +import jakarta.servlet.http.HttpServletRequest; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.SupportedDatabase; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.Path; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.view.ActionURL; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Stores basic site-wide configuration. + * @see org.labkey.api.settings.WriteableAppProps + */ +public interface AppProps +{ + AppProps _instance = new AppPropsImpl(); + + String SCOPE_SITE_SETTINGS = "SiteSettings"; + + // Used for all optional features; "experimental" for historical reasons. + String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; + String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. + String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; + String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; + String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; + String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; + String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; + + String UNKNOWN_VERSION = "Unknown Release Version"; + + static AppProps getInstance() + { + return _instance; + } + + static WriteableAppProps getWriteableInstance() + { + return new WriteableAppProps(ContainerManager.getRoot()); + } + + String getServerSessionGUID(); + + boolean isMailRecorderEnabled(); + + boolean isOptionalFeatureEnabled(String feature); + + boolean isDevMode(); + + @Nullable + String getEnlistmentId(); + + boolean isCachingAllowed(); + + boolean isRecompileJspEnabled(); + + /** + * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode + * without the risk of loading unwanted resources from a source tree that may not match the deployed server. + * + * WARNING: Setting this flag will interfere with the population of module beans, resulting in a + * mismatch between deployed modules and their properties on the server. + * + * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false + * + * @see org.labkey.api.module.DefaultModule#setSourcePath(String) + * @see org.labkey.api.module.DefaultModule#setBuildPath(String) + * @see DefaultModule#computeResourceDirectory() + */ + boolean isIgnoreModuleSource(); + + void setProjectRoot(String projectRoot); + + /** + * @return the root of the main source tree + */ + @Nullable + String getProjectRoot(); + + /** + * @return directory under which all containers will automatically have their own subdirectory for storing files + */ + @Nullable + File getFileSystemRoot(); + + @NotNull + UsageReportingLevel getUsageReportingLevel(); + + /** + * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". + * Or "Unknown Release Version". + */ + @NotNull + String getReleaseVersion(); + + /** + * Convenience method for getting the core schema version, returning 0.0 instead of null + */ + double getSchemaVersion(); + + String getContextPath(); + + Path getParsedContextPath(); + + int getServerPort(); + + String getScheme(); + + String getServerName(); + + /** + * Save the current request URL if the base server URL property is not set + */ + void ensureBaseServerUrl(HttpServletRequest request); + + void setContextPath(String contextPath); + + boolean isSetBaseServerUrl(); + + String getBaseServerUrl(); + + String getHomePageUrl(); + + ActionURL getHomePageActionURL(); + + String getSiteWelcomePageUrlString(); + + int getLookAndFeelRevision(); + + String getDefaultLsidAuthority(); + + String getPipelineToolsDirectory(); + + boolean isSSLRequired(); + + boolean isUserRequestedAdminOnlyMode(); + + String getAdminOnlyMessage(); + + boolean isShowRibbonMessage(); + + @Nullable String getRibbonMessage(); + + int getSSLPort(); + + int getMemoryUsageDumpInterval(); + + /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ + int getReadOnlyHttpRequestTimeout(); + + int getMaxBLOBSize(); + + boolean isExt3Required(); + + boolean isExt3APIRequired(); + + ExceptionReportingLevel getExceptionReportingLevel(); + + /** + * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full + * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. + * + * @return if navigation access is open + */ + boolean isNavigationAccessOpen(); + + boolean isSelfReportExceptions(); + + String getServerGUID(); + + String getBLASTServerBaseURL(); + + /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ + String getWebappConfigurationFilename(); + + /** + * Email address of the primary site or application administrator, set on the site settings page. Useful in error + * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., + * only impersonating troubleshooters). + * + * @return Email address of the primary site or application administrator + */ + @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); + + boolean isAllowApiKeys(); + + int getApiKeyExpirationSeconds(); + + boolean isAllowSessionKeys(); + + // configurable http security settings + + /** + * @return "SAMEORIGIN" or "DENY" or "ALLOW" + */ + String getXFrameOption(); + + String getStaticFilesPrefix(); + + boolean isWebfilesRootEnabled(); + + boolean isFileUploadDisabled(); + + boolean isInvalidFilenameUploadBlocked(); + + boolean isInvalidFilenameBlocked(); + + /** @return whether the server should include its name and version as a header in HTTP responses */ + boolean isIncludeServerHttpHeader(); + + /** + * @return List of configured external redirect hosts + */ + @NotNull + List getExternalRedirectHosts(); + + /** + * @return List of configured external resource hosts + */ + @Deprecated // Left for upgrade code only + @NotNull + List getExternalSourceHosts(); + + Map getStashedStartupProperties(); + + @NotNull String getDistributionName(); + + @NotNull String getDistributionFilename(); + + @NotNull Set getDistributionSupportedDatabases(); + + @NotNull List getAllowedExtensions(); + + @NotNull String getAllowedExternalResourceHosts(); +}