From 24b9f4b772ffebdd9e1b3d1fbb2a0847f59d8b39 Mon Sep 17 00:00:00 2001 From: Lakshmi Dintakurty Date: Mon, 24 Apr 2017 15:08:29 -0400 Subject: [PATCH 1/3] Cerner's initial performance changes with all loader classes refactored using dataSource instead of connection --- .gitignore | 1 + pom.xml | 40 +- .../CodeValidatorApiConfiguration.java | 454 ++- .../configuration/ConfiguredExpression.java | 16 +- .../configuration/ConfiguredValidator.java | 39 +- .../ValidationConfigurationLoader.java | 20 +- .../constants/VocabularyConstants.java | 1 + .../loader/BaseVocabularyLoader.java | 0 .../loader/VocabularyLoadRunner.java | 92 +- .../vocabularies/loader/VocabularyLoader.java | 5 +- .../vocabularies/loader/code/CdtLoader.java | 127 +- .../loader/code/CodeSystemOIDs.java | 6 +- .../vocabularies/loader/code/CptLoader.java | 40 +- .../loader/code/Icd10BaseLoader.java | 70 +- .../loader/code/Icd9BaseLoader.java | 37 +- .../loader/code/Icd9CmDxLoader.java | 1 - .../vocabularies/loader/code/IcdLoader.java | 2 + .../vocabularies/loader/code/LoincLoader.java | 140 +- .../loader/code/RxNormLoader.java | 68 +- .../loader/code/SnomedLoader.java | 59 +- .../loader/valueset/VsacLoader.java | 119 +- .../validation/NodeValidation.java | 5 +- .../validation/dao/CodeSystemCodeDAO.java | 202 ++ .../validation/dao/ValueSetDAO.java | 271 ++ .../validation/dto/NodeValidationResult.java | 18 + .../pool/AutoPilotObjectPoolInitializer.java | 31 + .../validation/pool/AutoPilotPool.java | 43 + .../validation/pool/AutoPilotWrapper.java | 62 + .../pool/AutoPilotWrapperFactory.java | 36 + .../validation/pool/AutoPilotWrapperPool.java | 17 + .../validation/services/ValidateRequest.java | 40 + .../validation/services/ValidateWorker.java | 127 + .../services/VocabularyValidationService.java | 369 ++- .../utils/ConfiguredExpressionFilter.java | 10 + .../ConfiguredExpressionLevelFilter.java | 130 + .../validation/utils/XpathUtils.java | 91 +- .../validation/validators/NodeValidator.java | 9 +- .../nodetypes/ClassCodeValidator.java | 64 +- .../nodetypes/CodeSystemCodeValidator.java | 110 +- ...eCodeNodeCountryCodeValuesetValidator.java | 61 +- ...CodeNodeLanguageCodeValuesetValidator.java | 62 +- ...mMatchesConfiguredCodeSystemValidator.java | 51 +- .../nodetypes/TextNodeValidator.java | 60 +- .../validators/nodetypes/UnitValidator.java | 52 +- .../nodetypes/ValueSetCodeValidator.java | 114 +- .../ValueSetNodeWithOnlyCodeValidator.java | 52 +- src/main/resources/CodeValidator.properties | 8 + src/main/resources/ais.properties | 8 + src/main/resources/schema.sql | 2 - .../ccda/validator/mu2ccdaDocumentSample.xml | 2741 +++++++++++++++++ .../ccda/validator/mu3ccdaDocumentSample.xml | 882 ++++++ .../ConfiguredExpressionLevelFilterTest.java | 57 + 52 files changed, 6363 insertions(+), 759 deletions(-) create mode 100644 src/main/java/org/sitenv/vocabularies/loader/BaseVocabularyLoader.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotObjectPoolInitializer.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotPool.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapper.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperFactory.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperPool.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionFilter.java create mode 100644 src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionLevelFilter.java create mode 100644 src/main/resources/CodeValidator.properties create mode 100644 src/main/resources/ais.properties create mode 100644 src/test/java/com/cerner/hs/ccda/validator/mu2ccdaDocumentSample.xml create mode 100644 src/test/java/com/cerner/hs/ccda/validator/mu3ccdaDocumentSample.xml create mode 100644 src/test/java/com/cerner/hs/ccda/validator/utils/ConfiguredExpressionLevelFilterTest.java diff --git a/.gitignore b/.gitignore index 764c55c..79fd5aa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /.settings /.classpath /.project +/target/ diff --git a/pom.xml b/pom.xml index cd40846..f7d07f9 100644 --- a/pom.xml +++ b/pom.xml @@ -2,8 +2,8 @@ 4.0.0 org.sitenv.vocabulary - codevalidator-api - 1.0.10 + cerner-codevalidator-api + 1.0.10-SNAPSHOT Code Validator API http://www.sitenv.org @@ -12,6 +12,16 @@ + + com.ximpleware + vtd-xml + 2.13 + + + commons-io + commons-io + 2.4 + junit junit @@ -111,11 +121,35 @@ apache-log4j-extras 1.2.17 + + junit + junit + 4.11 + + + org.springframework + spring-test + 4.1.6.RELEASE + + + + + com.ximpleware + vtd-xml + 2.13 + + + + org.apache.directory.studio + org.apache.commons.io + 2.4 + - codevalidator + + cerner-codevalidator-${version} org.apache.maven.plugins diff --git a/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java b/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java index 9c7b998..0a6a2c9 100644 --- a/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java +++ b/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java @@ -1,162 +1,346 @@ package org.sitenv.vocabularies.configuration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Executor; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathFactory; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Logger; import org.sitenv.vocabularies.loader.VocabularyLoadRunner; import org.sitenv.vocabularies.loader.VocabularyLoaderFactory; import org.sitenv.vocabularies.validation.NodeValidatorFactory; +import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.config.ServiceLocatorFactoryBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.core.env.Environment; +import org.springframework.core.io.Resource; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.jdbc.datasource.init.DataSourceInitializer; +import org.springframework.jdbc.datasource.init.DatabasePopulator; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; import org.springframework.orm.hibernate4.HibernateExceptionTranslator; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import org.springframework.oxm.jaxb.Jaxb2Marshaller; +import org.springframework.scheduling.annotation.AsyncConfigurer; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.transaction.PlatformTransactionManager; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPathFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - /** * Created by Brian on 2/5/2016. */ @Configuration +@EnableAsync @ComponentScan("org.sitenv.vocabularies") @EnableJpaRepositories("org.sitenv.vocabularies.validation.repositories") -public class CodeValidatorApiConfiguration { - - @Bean - public EntityManagerFactory entityManagerFactory() { - HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(false); - vendorAdapter.setShowSql(true); - LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); - factory.setJpaVendorAdapter(vendorAdapter); - factory.setPackagesToScan("org.sitenv.vocabularies.validation.entities"); - Properties jpaProperties = new Properties(); - jpaProperties.put("hibernate.hbm2ddl.auto", "none"); - jpaProperties.put("hibernate.dialect", "org.hibernate.dialect.HSQLDialect"); - jpaProperties.put("hibernate.format_sql", "true"); - jpaProperties.put("hibernate.show_sql", "false"); - factory.setDataSource(dataSource()); - factory.setJpaProperties(jpaProperties); - factory.afterPropertiesSet(); - return factory.getObject(); - } - - @Bean - public PlatformTransactionManager transactionManager() { - JpaTransactionManager txManager = new JpaTransactionManager(); - txManager.setEntityManagerFactory(entityManagerFactory()); - return txManager; - } - - @Bean - public HibernateExceptionTranslator hibernateExceptionTranslator() { - return new HibernateExceptionTranslator(); - } - - @Bean - public DataSource dataSource() { - return new EmbeddedDatabaseBuilder() - .setType(EmbeddedDatabaseType.HSQL) - .addScript("classpath:schema.sql") - .build(); - } - - @Bean - public static PropertySourcesPlaceholderConfigurer propertyPlaceholderConfigurer() { - PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer = new PropertySourcesPlaceholderConfigurer(); - propertySourcesPlaceholderConfigurer.setLocalOverride(true); - return propertySourcesPlaceholderConfigurer; - } - - @Bean - public ServiceLocatorFactoryBean vocabularyLoaderFactoryServiceLocatorFactoryBean() { - ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); - bean.setServiceLocatorInterface(VocabularyLoaderFactory.class); - return bean; - } - - @Bean - public VocabularyLoaderFactory vocabularyLoaderFactory() { - return (VocabularyLoaderFactory) vocabularyLoaderFactoryServiceLocatorFactoryBean().getObject(); - } - - @Bean - public ServiceLocatorFactoryBean vocabularyValidatorFactoryServiceLocatorFactoryBean() { - ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); - bean.setServiceLocatorInterface(NodeValidatorFactory.class); - return bean; - } - - @Bean - public NodeValidatorFactory vocabularyValidatorFactory() { - return (NodeValidatorFactory) vocabularyValidatorFactoryServiceLocatorFactoryBean().getObject(); - } - - @Autowired - @Bean - VocabularyLoadRunner vocabularyLoadRunner(final Environment environment, final VocabularyLoaderFactory vocabularyLoaderFactory, final DataSource dataSource){ - VocabularyLoadRunner vocabularyLoadRunner = null; - String localCodeRepositoryDir = environment.getProperty("vocabulary.localCodeRepositoryDir"); - String localValueSetRepositoryDir = environment.getProperty("vocabulary.localValueSetRepositoryDir"); - vocabularyLoadRunner = new VocabularyLoadRunner(); - System.out.println("LOADING VOCABULARY DATABASES FROM THE FOLLOWING RESOURCES: VALUESETS - " + localValueSetRepositoryDir + " CODES - " + localCodeRepositoryDir); - vocabularyLoadRunner.setCodeDirectory(localCodeRepositoryDir); - vocabularyLoadRunner.setValueSetDirectory(localValueSetRepositoryDir); - vocabularyLoadRunner.setDataSource(dataSource); - vocabularyLoadRunner.setVocabularyLoaderFactory(vocabularyLoaderFactory); - return vocabularyLoadRunner; - } - - @Bean - public List vocabularyValidationConfigurations(ValidationConfigurationLoader configurationLoader){ - return configurationLoader.getConfigurations().getExpressions(); - } - - @Bean - public DocumentBuilder documentBuilder() throws ParserConfigurationException { - DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); - domFactory.setNamespaceAware(true); - return domFactory.newDocumentBuilder(); - } - - @Bean - public XPathFactory xPathFactory(){ - return XPathFactory.newInstance(); - } - - @Autowired - @Bean - public ValidationConfigurationLoader validationConfigurationLoader(final Environment environment){ - ValidationConfigurationLoader validationConfigurationLoader = new ValidationConfigurationLoader(); - validationConfigurationLoader.setValidationConfigurationFilePath(environment.getProperty("referenceccda.configFile")); - validationConfigurationLoader.setUnmarshaller(castorMarshaller()); - return validationConfigurationLoader; - } - - @Bean - public Jaxb2Marshaller castorMarshaller() { - Jaxb2Marshaller jaxb2Marshaller = new Jaxb2Marshaller(); - jaxb2Marshaller.setPackagesToScan("org.sitenv.vocabularies.configuration"); - Map map = new HashMap<>(); - map.put("jaxb.formatted.output", true); - jaxb2Marshaller.setMarshallerProperties(map); - return jaxb2Marshaller; - } +@PropertySource("classpath:CodeValidator.properties") +public class CodeValidatorApiConfiguration implements AsyncConfigurer { + private static final Logger logger = Logger.getLogger(CodeValidatorApiConfiguration.class); + private static final String HSQL_JDBC_URL_TEMPLATE = "jdbc:h2:mem:inmemdb;DB_CLOSE_DELAY=-1;MULTI_THREADED=1;CACHE_SIZE=1048576"; + + + @Value("classpath:schema.sql") + private Resource HSQL_SCHEMA_SCRIPT; + + @Autowired + private Environment environment; + + @Value("${executor.maxPoolSize}") + private int maxPoolSize; + + @Value("${executor.corePoolSize}") + private int corePoolSize; + + @Value("${executor.queueCapacity}") + private int queueCapacity; + + @Bean(name = "taskExecutor") + public Executor getAsyncExecutor() { + ThreadPoolTaskExecutor e = new ThreadPoolTaskExecutor(); + e.setThreadGroupName("Spring-group-"); + e.setThreadNamePrefix("Spring-thread-"); + e.setMaxPoolSize(maxPoolSize); + e.setCorePoolSize(corePoolSize); + e.setQueueCapacity(queueCapacity); + e.initialize(); + return e; + } + + @Override + public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { + // TODO Auto-generated method stub + return null; + } + + @Bean + public ServiceLocatorFactoryBean vocabularyValidatorFactoryServiceLocatorFactoryBean() { + ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); + bean.setServiceLocatorInterface(NodeValidatorFactory.class); + return bean; + } + + @Bean + public NodeValidatorFactory vocabularyValidatorFactory() { + return (NodeValidatorFactory) vocabularyValidatorFactoryServiceLocatorFactoryBean().getObject(); + } + + @Bean + public EntityManagerFactory entityManagerFactory() { + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(false); + vendorAdapter.setShowSql(true); + LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); + factory.setJpaVendorAdapter(vendorAdapter); + factory.setPackagesToScan("org.sitenv.vocabularies.validation.entities"); + Properties jpaProperties = new Properties(); + jpaProperties.put("hibernate.hbm2ddl.auto", "none"); + // jpaProperties.put("hibernate.dialect", + // "org.hibernate.dialect.HSQLDialect"); + jpaProperties.put("hibernate.dialect", "org.hibernate.dialect.H2Dialect"); +// jpaProperties.put("hibernate.dialect", "org.hibernate.dialect.SQLServer2008Dialect"); + jpaProperties.put("hibernate.format_sql", "true"); + jpaProperties.put("hibernate.show_sql", "false"); + jpaProperties.put("hibernate.connection.pool_size", "80"); + factory.setDataSource(dataSource()); + factory.setJpaProperties(jpaProperties); + factory.afterPropertiesSet(); + return factory.getObject(); + } + + @Bean + public PlatformTransactionManager transactionManager() { + JpaTransactionManager txManager = new JpaTransactionManager(); + txManager.setEntityManagerFactory(entityManagerFactory()); + return txManager; + } + + @Bean + public HibernateExceptionTranslator hibernateExceptionTranslator() { + return new HibernateExceptionTranslator(); + } + + @Autowired + @Bean + public DataSourceInitializer dataSourceInitializer(final DataSource dataSource) { + final DataSourceInitializer initializer = new DataSourceInitializer(); + initializer.setDataSource(dataSource); + initializer.setDatabasePopulator(databasePopulator()); + return initializer; + } + + private DatabasePopulator databasePopulator() { + final ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); + populator.addScript(HSQL_SCHEMA_SCRIPT); + return populator; + } + + @Value("${dataSource.initialSize}") + private int initialSize; + + @Value("${dataSource.minIdle}") + private int minIdle; + + @Value("${dataSource.maxActive}") + private int maxActive; + + @Bean + public DataSource dataSource() { + BasicDataSource ds = new BasicDataSource(); + ds.setUrl(HSQL_JDBC_URL_TEMPLATE); + ds.setUsername("sa"); + ds.setPassword(""); + ds.setInitialSize(initialSize); + ds.setMinIdle(minIdle); + ds.setMaxTotal(maxActive); // DBCP2 maxTotal: The maximum number of active connections that can be allocated from this pool at the same time, or negative for no limit. + // ds.setDriverClassName("org.hsqldb.jdbcDriver"); + ds.setDriverClassName("org.h2.Driver"); + //ds.setDriverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + return ds; + } + + @Bean + public static PropertySourcesPlaceholderConfigurer propertyPlaceholderConfigurer() { + PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer = new PropertySourcesPlaceholderConfigurer(); + propertySourcesPlaceholderConfigurer.setLocalOverride(true); + return propertySourcesPlaceholderConfigurer; + } + + @Bean + public ServiceLocatorFactoryBean vocabularyLoaderFactoryServiceLocatorFactoryBean() { + ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); + bean.setServiceLocatorInterface(VocabularyLoaderFactory.class); + return bean; + } + + @Bean + public VocabularyLoaderFactory vocabularyLoaderFactory() { + return (VocabularyLoaderFactory) vocabularyLoaderFactoryServiceLocatorFactoryBean().getObject(); + } + + // @Bean + // public ServiceLocatorFactoryBean + // vocabularyValidatorFactoryServiceLocatorFactoryBean() { + // ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); + // bean.setServiceLocatorInterface(VocabularyValidatorFactory.class); + // return bean; + // } + + // @Bean + // public VocabularyValidatorFactory vocabularyValidatorFactory() { + // return (VocabularyValidatorFactory) + // vocabularyValidatorFactoryServiceLocatorFactoryBean().getObject(); + // } + + @Autowired + @Bean + VocabularyLoadRunner vocabularyLoadRunner(final Environment environment, + final VocabularyLoaderFactory vocabularyLoaderFactory, final DataSourceInitializer dataSourceInitializer, + final DataSource dataSource) { + // VocabularyLoadRunner vocabularyLoadRunner(final Environment + // environment, final VocabularyLoaderFactory vocabularyLoaderFactory, + // final DataSourceInitializer dataSourceInitializer, final DataSource + // dataSource){ + VocabularyLoadRunner vocabularyLoadRunner = null; + String localCodeRepositoryDir = environment.getProperty("vocabulary.localCodeRepositoryDir"); + String localValueSetRepositoryDir = environment.getProperty("vocabulary.localValueSetRepositoryDir"); + vocabularyLoadRunner = new VocabularyLoadRunner(); + logger.info("LOADING VOCABULARY DATABASES FROM THE FOLLOWING RESOURCES: VALUESETS - " + + localValueSetRepositoryDir + " CODES - " + localCodeRepositoryDir); + vocabularyLoadRunner.setCodeDirectory(localCodeRepositoryDir); + vocabularyLoadRunner.setValueSetDirectory(localValueSetRepositoryDir); + vocabularyLoadRunner.setDataSource(dataSource); + vocabularyLoadRunner.setVocabularyLoaderFactory(vocabularyLoaderFactory); + + BasicConfigurator.configure(); + + return vocabularyLoadRunner; + } + + /* + * Following open source method is commented and re-factored it to support + * MU2 document validation. + */ + + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + /* + * @Bean public List + * vocabularyValidationConfigurations(ValidationConfigurationLoader + * configurationLoader){ return + * configurationLoader.getConfigurations().getExpressions(); } + */ + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + // @Bean + // public DocumentBuilder documentBuilder() throws + // ParserConfigurationException { + // DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); + // domFactory.setNamespaceAware(true); + // return domFactory.newDocumentBuilder(); + // } + + @Bean + public DocumentBuilderFactory documentBuilderFactory() throws ParserConfigurationException { + DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); + domFactory.setNamespaceAware(true); + return domFactory; + } + + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + @Bean + public XPathFactory xPathFactory() { + return XPathFactory.newInstance(); + } + + @Autowired + @Bean + public ValidationConfigurationLoader validationConfigurationLoader(final Environment environment) { + ValidationConfigurationLoader validationConfigurationLoader = new ValidationConfigurationLoader(); + validationConfigurationLoader + .setValidationConfigurationFilePath(environment.getProperty("referenceccda.configFile")); + validationConfigurationLoader.setUnmarshaller(castorMarshaller()); + return validationConfigurationLoader; + } + + @Bean + public Jaxb2Marshaller castorMarshaller() { + Jaxb2Marshaller jaxb2Marshaller = new Jaxb2Marshaller(); + jaxb2Marshaller.setPackagesToScan("org.sitenv.vocabularies.configuration"); + Map map = new HashMap<>(); + map.put("jaxb.formatted.output", true); + jaxb2Marshaller.setMarshallerProperties(map); + return jaxb2Marshaller; + } + + /* + * Following set of code is added for enhancing open source code base to + * support MU2 document validation. + */ + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + + @Bean + public List vocabularyValidationConfigurations( + ValidationConfigurationLoader configurationLoader) { + + List mu3Expressions = new ArrayList(); + List expressions = configurationLoader.getConfigurations().getExpressions(); + for (ConfiguredExpression configuredExpression : expressions) { + List configuredValidators = configuredExpression.getConfiguredValidators(); + for (ConfiguredValidator configuredValidator : configuredValidators) { + if (StringUtils.isEmpty(configuredValidator.getScope())) { + mu3Expressions.add(configuredExpression); + break; + } + } + } + logger.info("Number of MU3 expressions:" + mu3Expressions.size()); + return mu3Expressions; + } + + @Bean + public List vocabularyValidationConfigurationsForMu2( + ValidationConfigurationLoader configurationLoader) { + + List mu2Expressions = new ArrayList(); + List expressions = configurationLoader.getConfigurations().getExpressions(); + for (ConfiguredExpression configuredExpression : expressions) { + List configuredValidators = configuredExpression.getConfiguredValidators(); + for (ConfiguredValidator configuredValidator : configuredValidators) { + if (configuredValidator.getScope() != null && configuredValidator.getScope().equals("1.1")) { + mu2Expressions.add(configuredExpression); + break; + } + } + } + logger.info("Number of MU2 expressions:" + mu2Expressions.size()); + return mu2Expressions; + } + + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- } diff --git a/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredExpression.java b/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredExpression.java index c27c342..391b96b 100644 --- a/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredExpression.java +++ b/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredExpression.java @@ -1,7 +1,17 @@ package org.sitenv.vocabularies.configuration; -import javax.xml.bind.annotation.*; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import com.ximpleware.AutoPilot; +import com.ximpleware.XPathParseException; /** * Created by Brian on 2/10/2016. @@ -13,11 +23,11 @@ public class ConfiguredExpression { String configuredXpathExpression; @XmlElement(name = "validator") List configuredValidators = null; - + public String getConfiguredXpathExpression() { return configuredXpathExpression; } - + public void setConfiguredXpathExpression(String configuredXpathExpression) { this.configuredXpathExpression = configuredXpathExpression; } diff --git a/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredValidator.java b/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredValidator.java index 6cf4c36..400e350 100644 --- a/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredValidator.java +++ b/src/main/java/org/sitenv/vocabularies/configuration/ConfiguredValidator.java @@ -2,6 +2,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -11,7 +12,17 @@ @XmlRootElement(name = "validator") @XmlAccessorType(XmlAccessType.FIELD) public class ConfiguredValidator { - @XmlElement(name = "name") + /* + * Following XML attribute 'scope' is added to support MU2 document validation. + * Xpath configuration validator having scope attribute set to '1.1' are targeted for + * CCDA R 1.1 CCDA document validation. + */ + //------------------------- INTERNAL CODE CHAGNE START -------------------------- + @XmlAttribute(name = "scope") + String scope; + //------------------------- INTERNAL CODE CHAGNE END -------------------------- + + @XmlElement(name = "name") String name; @XmlElement(name = "nodeType") String nodeType; @@ -21,7 +32,9 @@ public class ConfiguredValidator { String allowedValuesetOids; @XmlElement(name = "allowedCodesystemNames") String allowedCodesystemNames; - + @XmlAttribute(name = "id") + String id; + public String getName() { return name; } @@ -61,5 +74,27 @@ public String getAllowedCodesystemNames() { public void setAllowedCodesystemNames(String allowedCodesystemNames) { this.allowedCodesystemNames = allowedCodesystemNames; } + + /* + * setter and getter methods for 'scope' attribute to support MU2 document validation. + * scope attribute set to '1.1' are targeted for CCDA R 1.1 CCDA document validation. + */ + //------------------------- INTERNAL CODE CHAGNE START -------------------------- + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + //------------------------- INTERNAL CODE CHAGNE END -------------------------- } diff --git a/src/main/java/org/sitenv/vocabularies/configuration/ValidationConfigurationLoader.java b/src/main/java/org/sitenv/vocabularies/configuration/ValidationConfigurationLoader.java index 0aca331..eaf60f7 100644 --- a/src/main/java/org/sitenv/vocabularies/configuration/ValidationConfigurationLoader.java +++ b/src/main/java/org/sitenv/vocabularies/configuration/ValidationConfigurationLoader.java @@ -1,16 +1,20 @@ package org.sitenv.vocabularies.configuration; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.oxm.Unmarshaller; - -import javax.xml.transform.stream.StreamSource; import java.io.FileInputStream; import java.io.IOException; +import javax.xml.transform.stream.StreamSource; + +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.validation.pool.AutoPilotObjectPoolInitializer; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.oxm.Unmarshaller; + /** * Created by Brian on 2/10/2016. */ public class ValidationConfigurationLoader implements InitializingBean { + private static Logger logger = Logger.getLogger(ValidationConfigurationLoader.class); private Unmarshaller unmarshaller; private String validationConfigurationFilePath; private Configurations configurations; @@ -32,7 +36,9 @@ public Object xmlToObject(String fileName) throws IOException { FileInputStream fis = null; try { fis = new FileInputStream(fileName); - return configurations = (Configurations) unmarshaller.unmarshal(new StreamSource(fis)); + configurations = (Configurations) unmarshaller.unmarshal(new StreamSource(fis)); + logger.info("Loading expressions from: " + fileName + ". # Expressions:" + configurations.getExpressions().size()); + return configurations; } finally { fis.close(); } @@ -41,5 +47,9 @@ public Object xmlToObject(String fileName) throws IOException { @Override public void afterPropertiesSet() throws Exception { xmlToObject(validationConfigurationFilePath); + // ------------------------- INTERNAL CODE CHANGE START -------------------------- + AutoPilotObjectPoolInitializer initializer = new AutoPilotObjectPoolInitializer(); + initializer.initFromExpressions(configurations.getExpressions()); + // ------------------------- INTERNAL CODE CHANGE END -------------------------- } } \ No newline at end of file diff --git a/src/main/java/org/sitenv/vocabularies/constants/VocabularyConstants.java b/src/main/java/org/sitenv/vocabularies/constants/VocabularyConstants.java index b760030..ac29fd3 100644 --- a/src/main/java/org/sitenv/vocabularies/constants/VocabularyConstants.java +++ b/src/main/java/org/sitenv/vocabularies/constants/VocabularyConstants.java @@ -1,5 +1,6 @@ package org.sitenv.vocabularies.constants; +import java.util.HashMap; import java.util.Map; import java.util.TreeMap; diff --git a/src/main/java/org/sitenv/vocabularies/loader/BaseVocabularyLoader.java b/src/main/java/org/sitenv/vocabularies/loader/BaseVocabularyLoader.java new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java index ae0ba98..2461157 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java +++ b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java @@ -1,16 +1,23 @@ package org.sitenv.vocabularies.loader; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.InitializingBean; - -import javax.sql.DataSource; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.util.Arrays; +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.EntityTransaction; +import javax.sql.DataSource; + +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; + /** * Created by Brian on 2/6/2016. */ @@ -22,19 +29,40 @@ public class VocabularyLoadRunner implements InitializingBean, DisposableBean { private boolean recursive = true; private DataSource dataSource; - public void loadDirectory(String directory, Connection connection) throws IOException { + @Autowired + ValueSetDAO vsdao; + + @Autowired + CodeSystemCodeDAO csdao; + +// @Autowired +// EntityManagerFactory entityManagerFactory; + + public void loadDirectory(String directory, DataSource datasource) throws IOException { File dir = new File(directory); if (dir.isFile()) { throw new IOException("Directory to Load is a file and not a directory"); } else { File[] list = dir.listFiles(); for (File file : list) { - load(file, connection); + load(file, datasource); } } } - - private void load(File directory, Connection connection) throws IOException { + +// public void loadDirectory(String directory, Connection connection) throws IOException { +// File dir = new File(directory); +// if (dir.isFile()) { +// throw new IOException("Directory to Load is a file and not a directory"); +// } else { +// File[] list = dir.listFiles(); +// for (File file : list) { +// load(file, connection); +// } +// } +// } + + private void load(File directory, DataSource datasource) throws IOException { if (directory.isDirectory() && !directory.isHidden()) { File[] filesToLoad = directory.listFiles(); logger.debug("Building Loader for directory: " + directory.getName() + "..."); @@ -42,13 +70,29 @@ private void load(File directory, Connection connection) throws IOException { if (loader != null && filesToLoad != null) { logger.debug("Loader built..."); logger.info("Loading files in : " + directory.getName() + "..."); - loader.load(Arrays.asList(filesToLoad), connection); - logger.debug("File loaded..."); + long n = loader.load(Arrays.asList(filesToLoad), datasource); + logger.info("File loaded...(" + n + ") entries."); } else { logger.debug("Building of Loader Failed."); } } } + +// private void load(File directory, Connection connection) throws IOException { +// if (directory.isDirectory() && !directory.isHidden()) { +// File[] filesToLoad = directory.listFiles(); +// logger.debug("Building Loader for directory: " + directory.getName() + "..."); +// VocabularyLoader loader = vocabularyLoaderFactory.getVocabularyLoader(directory.getName()); +// if (loader != null && filesToLoad != null) { +// logger.debug("Loader built..."); +// logger.info("Loading files in : " + directory.getName() + "..."); +// loader.load(Arrays.asList(filesToLoad), connection); +// logger.debug("File loaded..."); +// } else { +// logger.debug("Building of Loader Failed."); +// } +// } +// } public String getCodeDirectory() { return codeDirectory; @@ -90,24 +134,44 @@ public void afterPropertiesSet() throws Exception { connection.setAutoCommit(false); if (codeDirectory != null && !codeDirectory.trim().equals("")) { logger.info("Loading vocabularies at: " + codeDirectory + "..."); - loadDirectory(codeDirectory, connection); +// loadDirectory(codeDirectory, connection); + loadDirectory(codeDirectory, dataSource); logger.info("Vocabularies loaded..."); } connection.commit(); if (valueSetDirectory != null && !valueSetDirectory.trim().equals("")) { logger.info("Loading value sets at: " + valueSetDirectory + "..."); - loadDirectory(valueSetDirectory, connection); +// loadDirectory(valueSetDirectory, connection); + loadDirectory(valueSetDirectory, dataSource); logger.info("Value Sets loaded..."); } connection.commit(); logger.info("!!!!*********** VOCABULARY DATABASE HAS FINISHED LOADING - SERVER WILL CONTINUE AND SHOULD BE DONE SHORTLY. ***********!!!!"); + + vsdao.loadValueSets(dataSource); + connection.commit(); + csdao.loadCodes(dataSource); + connection.commit(); + + logger.info("!!!!*********** INITIALIZED HASH SETS. ***********!!!!"); + +// entityManagerFactory.getCache().evictAll(); +// EntityManager em = entityManagerFactory.createEntityManager(); +// EntityTransaction tx = em.getTransaction(); +// tx.begin(); +// em.flush(); +// em.clear(); +// tx.commit(); +// +// logger.info("!!!!*********** EM Evicted All. ***********!!!!"); + } catch (Exception e) { logger.error("Failed to load configured vocabulary directory.", e); }finally { try { - if(connection != null || !(connection.isClosed())) { + if(connection != null && !(connection.isClosed())) { connection.close(); } } catch (SQLException e) { diff --git a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java index 1d1610d..95a97e8 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java @@ -4,9 +4,12 @@ import java.sql.Connection; import java.util.List; +import javax.sql.DataSource; + /** * Created by Brian on 2/7/2016. */ public interface VocabularyLoader { - void load(List file, Connection connection); +// void load(List file, Connection connection); + long load(List file, DataSource ds); } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java index 28f0cbd..ef01088 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java @@ -1,5 +1,16 @@ package org.sitenv.vocabularies.loader.code; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +import javax.sql.DataSource; + import org.apache.commons.lang3.text.StrBuilder; import org.apache.log4j.Logger; import org.apache.poi.ss.usermodel.Cell; @@ -8,24 +19,21 @@ import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.sitenv.vocabularies.loader.BaseCodeLoader; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; - @Component(value = "CDT") - public class CdtLoader extends BaseCodeLoader { +public class CdtLoader extends BaseCodeLoader { private static Logger logger = Logger.getLogger(CdtLoader.class); - @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource ds) { + long n = 0; StrBuilder insertQueryBuilder = null; String insertQueryPrefix = codeTableInsertSQLPrefix; + JdbcTemplate t = new JdbcTemplate(ds); for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { String codeSystem = file.getParentFile().getName(); @@ -54,15 +62,31 @@ public void load(List filesToLoad, Connection connection) { code = codeCell.getStringCellValue(); displayName = descriptionCell.getStringCellValue(); - buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, CodeSystemOIDs.CDT.codesystemOID()); +// insertQueryBuilder.append("("); +// insertQueryBuilder.append("'"); +// insertQueryBuilder.append(code.toUpperCase().trim()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(displayName.replaceAll("'", "''").toUpperCase().trim()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("')"); + + n++; + buildCodeInsertQueryString(insertQueryBuilder, code.toUpperCase(), displayName.toUpperCase(), codeSystem, CodeSystemOIDs.CDT.codesystemOID()); + t.update(insertQueryPrefix.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + +// t.update(insertQueryPrefix,code.toUpperCase().trim(),displayName.toUpperCase().trim(),file.getParentFile().getName(),CodeSystemOIDs.CDT.codesystemOID()); + } } - insertCode(insertQueryBuilder.toString(), connection); + //insertCode(insertQueryBuilder.toString(), connection); } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { try { if(workBook != null) { @@ -77,8 +101,81 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } +// public void load(List filesToLoad, Connection connection) { +// StrBuilder insertQueryBuilder = null; +//// String insertQueryPrefix = "insert into CODES (ID, CODE, DISPLAYNAME, CODESYSTEM) values "; +// String insertQueryPrefix = "insert into CODES ( CODE, DISPLAYNAME, CODESYSTEM) values "; +// for (File file : filesToLoad) { +// if (file.isFile() && !file.isHidden()) { +// InputStream inputStream = null; +// XSSFWorkbook workBook = null; +// try { +// logger.debug("Loading CDT File: " + file.getName()); +// inputStream = new FileInputStream(file); +// workBook = new XSSFWorkbook(inputStream); +// +// for (int i = 0; i < workBook.getNumberOfSheets(); i++) { +// insertQueryBuilder = new StrBuilder(insertQueryPrefix); +// XSSFSheet sheet = workBook.getSheetAt(i); +// +// for (int count = 26; count < sheet.getLastRowNum()+1; count++) { +// if (!isRowEmpty(sheet.getRow(count))) { +// String code; +// String displayName; +// +// XSSFCell codeCell = sheet.getRow(count).getCell(0); +// XSSFCell descriptionCell = sheet.getRow(count).getCell(2); +// +// codeCell.setCellType(Cell.CELL_TYPE_STRING); +// descriptionCell.setCellType(Cell.CELL_TYPE_STRING); +// +// +// code = codeCell.getStringCellValue(); +// displayName = descriptionCell.getStringCellValue(); +// +// if(count > 26){ +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +//// insertQueryBuilder.append("DEFAULT"); +//// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append("'"); +// insertQueryBuilder.append(code.toUpperCase().trim()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(displayName.replaceAll("'", "''").toUpperCase().trim()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("')"); +// +//// CcdaCodeSystemCodeValidatorDAO.addRow(code.toUpperCase().trim(),file.getParentFile().getName(),displayName.toUpperCase().trim()); +// +// } +// } +// doInsert(insertQueryBuilder.toString(), connection); +// } +// } catch (IOException e) { +// logger.error(e); +// } catch (SQLException e) { +// e.printStackTrace(); +// } finally { +// try { +// if(workBook != null) { +// workBook.close(); +// } +// if(inputStream != null) { +// inputStream.close(); +// } +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// } +// } +// } + public static boolean isRowEmpty(Row row) { if(row != null){ for (int c = row.getFirstCellNum(); c < row.getLastCellNum(); c++) { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/CodeSystemOIDs.java b/src/main/java/org/sitenv/vocabularies/loader/code/CodeSystemOIDs.java index 1c3ae19..943e773 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/CodeSystemOIDs.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/CodeSystemOIDs.java @@ -4,9 +4,9 @@ * Created by Brian on 8/9/2016. */ public enum CodeSystemOIDs { - SNOMEDCT("2.16.840.1.113883.6.96"), - RXNORM("2.16.840.1.113883.6.88"), - LOINC("2.16.840.1.113883.6.1"), + SNOMEDCT("2.16.840.1.113883.6.96"), + RXNORM("2.16.840.1.113883.6.88"), + LOINC("2.16.840.1.113883.6.1"), ICD10CM("2.16.840.1.113883.6.90"), ICD10PCS("2.16.840.1.113883.6.4"), CPT4("2.16.840.1.113883.6.12"), diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java index d4b8715..a23eb18 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java @@ -13,6 +13,9 @@ import java.sql.SQLException; import java.util.List; +import javax.sql.DataSource; +import org.springframework.jdbc.core.JdbcTemplate; + /** * Created by Brian on 2/7/2016. */ @@ -26,13 +29,14 @@ public CptLoader() { } @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource ds) { + long n = 0; BufferedReader br = null; FileReader fileReader = null; try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); int totalCount = 0, pendingCount = 0; - + JdbcTemplate t = new JdbcTemplate(ds); for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { logger.debug("Loading CPT File: " + file.getName()); @@ -46,24 +50,30 @@ public void load(List filesToLoad, Connection connection) { String displayName = line.substring(line.indexOf(" ")); buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, oid); - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } +// if ((++totalCount % BATCH_SIZE) == 0) { +// insertCode(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(codeTableInsertSQLPrefix); +// pendingCount = 0; +// } + + n++; + t.update(codeTableInsertSQLPrefix,code.toUpperCase().trim(),displayName.toUpperCase().trim(),codeSystem,CodeSystemOIDs.CDT.codesystemOID()); + } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// insertCode(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); - } finally { + } +// catch (SQLException e) { +// e.printStackTrace(); +// } + finally { if (br != null) { try { fileReader.close(); @@ -73,5 +83,7 @@ public void load(List filesToLoad, Connection connection) { } } } + + return n; } } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java index 7084299..39915c7 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java @@ -1,8 +1,5 @@ package org.sitenv.vocabularies.loader.code; -import org.apache.commons.lang3.text.StrBuilder; -import org.apache.log4j.Logger; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -11,6 +8,13 @@ import java.sql.SQLException; import java.util.List; +import javax.sql.DataSource; + +import org.apache.commons.lang3.text.StrBuilder; +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.springframework.jdbc.core.JdbcTemplate; + /** * Created by Brian on 2/7/2016. */ @@ -18,8 +22,9 @@ public abstract class Icd10BaseLoader extends IcdLoader { private static Logger logger = Logger.getLogger(Icd10BaseLoader.class); protected String oid; - @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource datasource) { + long n = 0; + JdbcTemplate t = new JdbcTemplate(datasource); BufferedReader br = null; FileReader fileReader = null; try { @@ -37,27 +42,59 @@ public void load(List filesToLoad, Connection connection) { String code = buildDelimitedIcdCode(available.substring(6, 13)); String shortDisplayName = available.substring(16, 77); String longDisplayName = available.substring(77); +// if (pendingCount++ > 0) { +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +// insertQueryBuilder.append("DEFAULT"); +// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append(buildDelimitedIcdCode(available.substring(6, 13).trim()).toUpperCase()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(available.substring(77).trim().toUpperCase().replaceAll("'", "''")); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(oid); +// insertQueryBuilder.append("')"); + + n++; + //@TODO Merge to update statements into single update statement ????? + //@TODO Merge to update statements into single update statement ????? + + buildCodeInsertQueryString(insertQueryBuilder, code, shortDisplayName, codeSystem, oid); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + buildCodeInsertQueryString(insertQueryBuilder, code, longDisplayName, codeSystem, oid); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + +// t.update(codeTableInsertSQLPrefix,buildDelimitedIcdCode(code.trim()).toUpperCase(),longDisplayName.trim().toUpperCase(),file.getParentFile().getName(),oid); +// t.update(codeTableInsertSQLPrefix,buildDelimitedIcdCode(code.trim()).toUpperCase(),shortDisplayName.trim().toUpperCase(),file.getParentFile().getName(),oid); - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } +// t.update(insertQueryPrefix,buildDelimitedIcdCode(available.substring(6, 13).trim()).toUpperCase(),available.substring(77).trim().toUpperCase(),file.getParentFile().getName(),oid); + +// if ((++totalCount % 2500) == 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(insertQueryPrefix); +// pendingCount = 0; +// } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { if (br != null) { try { @@ -68,5 +105,6 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java index be5554e..a268040 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java @@ -3,6 +3,8 @@ import org.apache.commons.lang3.text.StrBuilder; import org.apache.log4j.Logger; import org.sitenv.vocabularies.loader.BaseCodeLoader; +import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; +import org.springframework.jdbc.core.JdbcTemplate; import java.io.BufferedReader; import java.io.File; @@ -12,6 +14,8 @@ import java.sql.SQLException; import java.util.List; +import javax.sql.DataSource; + import static org.sitenv.vocabularies.loader.code.IcdLoader.buildDelimitedIcdCode; /** @@ -22,9 +26,11 @@ public abstract class Icd9BaseLoader extends BaseCodeLoader { protected String oid; @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource datasource) { + long n = 0; BufferedReader br = null; FileReader fileReader = null; + JdbcTemplate t = new JdbcTemplate(datasource); try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); int totalCount = 0, pendingCount = 0; @@ -41,24 +47,30 @@ public void load(List filesToLoad, Connection connection) { String code = buildDelimitedIcdCode(line.substring(0, 5)); String displayName = line.substring(6); buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, oid); +// +// if ((++totalCount % BATCH_SIZE) == 0) { +// insertCode(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(codeTableInsertSQLPrefix); +// pendingCount = 0; +// } + n++; + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); +// t.update(codeTableInsertSQLPrefix,code.trim().toUpperCase(),displayName.trim().toUpperCase(),codeSystem,oid); - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// insertCode(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { if (br != null) { try { @@ -69,5 +81,6 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9CmDxLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9CmDxLoader.java index 8704c6a..3a26492 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9CmDxLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9CmDxLoader.java @@ -11,5 +11,4 @@ public class Icd9CmDxLoader extends Icd9BaseLoader { public Icd9CmDxLoader() { this.oid = CodeSystemOIDs.ICD9CMDX.codesystemOID(); } - } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java index bfbc4e4..6ebd623 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java @@ -44,4 +44,6 @@ protected static String buildDelimitedIcdCode(String code) { return builder.build(); } + +// protected abstract void setOID(String oid); } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java index 2a3900e..7fb6d48 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java @@ -5,6 +5,8 @@ import org.apache.log4j.Logger; import org.sitenv.vocabularies.loader.BaseCodeLoader; import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import java.io.BufferedReader; @@ -15,6 +17,8 @@ import java.sql.SQLException; import java.util.List; +import javax.sql.DataSource; + /** * Created by Brian on 2/7/2016. */ @@ -22,8 +26,9 @@ public class LoincLoader extends BaseCodeLoader implements VocabularyLoader { private static Logger logger = Logger.getLogger(LoincLoader.class); - @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource datasource) { + long n = 0; + JdbcTemplate t = new JdbcTemplate(datasource); BufferedReader br = null; FileReader fileReader = null; try { @@ -42,6 +47,20 @@ public void load(List filesToLoad, Connection connection) { continue; // skip header row } else { String[] line = available.replaceAll("^\"", "").split("\"?(,|$)(?=(([^\"]*\"){2})*[^\"]*$) *\"?"); +// if (pendingCount++ > 0) { +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +// insertQueryBuilder.append("DEFAULT"); +// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append(StringUtils.strip(line[0], "\"").toUpperCase()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(StringUtils.strip(line[29], "\"").toUpperCase().replaceAll("'", "''")); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(CodeSystemOIDs.LOINC.codesystemOID()); +// insertQueryBuilder.append("')"); String code = StringUtils.strip(line[0], "\""); String codeSystem = file.getParentFile().getName(); @@ -49,28 +68,48 @@ public void load(List filesToLoad, Connection connection) { String longCommonName = StringUtils.strip(line[29], "\""); String componentName = StringUtils.strip(line[1], "\""); String shortName = StringUtils.strip(line[23], "\""); + n++; + + buildCodeInsertQueryString(insertQueryBuilder, code, longCommonName.toUpperCase(), codeSystem, oid); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + + buildCodeInsertQueryString(insertQueryBuilder, code, componentName.toUpperCase(), codeSystem, oid); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + + + buildCodeInsertQueryString(insertQueryBuilder, code, shortName.toUpperCase(), codeSystem, oid); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + + +// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),longCommonName.toUpperCase(),codeSystem,oid); +// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),componentName.toUpperCase(),codeSystem,oid); +// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),shortName.toUpperCase(),codeSystem,oid); - buildCodeInsertQueryString(insertQueryBuilder, code, longCommonName, codeSystem, oid); - buildCodeInsertQueryString(insertQueryBuilder, code, componentName, codeSystem, oid); - buildCodeInsertQueryString(insertQueryBuilder, code, shortName, codeSystem, oid); +// t.update(insertQueryPrefix,StringUtils.strip(line[0], "\"").toUpperCase(),StringUtils.strip(line[29], "\"").toUpperCase(),file.getParentFile().getName(),CodeSystemOIDs.LOINC.codesystemOID()); - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } +// if ((++totalCount % 5000) == 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(insertQueryPrefix); +// pendingCount = 0; +// } } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { if (br != null) { try { @@ -81,6 +120,75 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } +// @Override +// public void load(List filesToLoad, Connection connection) { +// BufferedReader br = null; +// FileReader fileReader = null; +// try { +//// String insertQueryPrefix = "insert into CODES (ID, CODE, DISPLAYNAME, CODESYSTEM) values "; +// String insertQueryPrefix = "insert into CODES ( CODE, DISPLAYNAME, CODESYSTEM) values "; +// StrBuilder insertQueryBuilder = new StrBuilder(insertQueryPrefix); +// int totalCount = 0, pendingCount = 0; +// +// for (File file : filesToLoad) { +// if (file.isFile() && !file.isHidden()) { +// logger.debug("Loading LOINC File: " + file.getName()); +// int count = 0; +// fileReader = new FileReader(file); +// br = new BufferedReader(fileReader); +// String available; +// while ((available = br.readLine()) != null) { +// if ((count++ == 0)) { +// continue; // skip header row +// } else { +// String[] line = StringUtils.splitPreserveAllTokens(available, ",", 3); +// if (pendingCount++ > 0) { +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +//// insertQueryBuilder.append("DEFAULT"); +//// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append("'"); +// insertQueryBuilder.append(StringUtils.strip(line[0], "\"").toUpperCase()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(StringUtils.strip(line[1], "\"").toUpperCase().replaceAll("'", "''")); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("')"); +// +//// CcdaCodeSystemCodeValidatorDAO.addRow(StringUtils.strip(line[0], "\"").toUpperCase(), file.getParentFile().getName(), +//// StringUtils.strip(line[1], "\"").toUpperCase()); +// +// if ((++totalCount % 900) == 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(insertQueryPrefix); +// pendingCount = 0; +// } +// } +// } +// } +// } +// if (pendingCount > 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// } +// } catch (IOException e) { +// logger.error(e); +// } catch (SQLException e) { +// e.printStackTrace(); +// } finally { +// if (br != null) { +// try { +// fileReader.close(); +// br.close(); +// } catch (IOException e) { +// logger.error(e); +// } +// } +// } +// } } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java index 0fc4513..c218e3d 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java @@ -1,20 +1,21 @@ package org.sitenv.vocabularies.loader.code; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.List; + +import javax.sql.DataSource; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.text.StrBuilder; import org.apache.log4j.Logger; import org.sitenv.vocabularies.loader.BaseCodeLoader; import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; - /** * Created by Brian on 2/7/2016. */ @@ -22,8 +23,9 @@ public class RxNormLoader extends BaseCodeLoader implements VocabularyLoader { private static Logger logger = Logger.getLogger(RxNormLoader.class); - @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource datasource) { + long n = 0; + JdbcTemplate t = new JdbcTemplate(datasource); FileReader fileReader = null; BufferedReader br = null; try { @@ -41,25 +43,45 @@ public void load(List filesToLoad, Connection connection) { String[] line = StringUtils.splitPreserveAllTokens(available, "|", 16); String code = line[0]; String displayName = line[14]; +// if (pendingCount++ > 0) { +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +// insertQueryBuilder.append("DEFAULT"); +// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append(line[0]); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(line[14].toUpperCase().replaceAll("'", "''")); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(CodeSystemOIDs.RXNORM.codesystemOID()); +// insertQueryBuilder.append("')"); - buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, CodeSystemOIDs.RXNORM.codesystemOID()); + n++; + buildCodeInsertQueryString(insertQueryBuilder, code, displayName.toUpperCase(), codeSystem, CodeSystemOIDs.RXNORM.codesystemOID()); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + +// t.update(codeTableInsertSQLPrefix,code,displayName.toUpperCase(),codeSystem,CodeSystemOIDs.RXNORM.codesystemOID()); - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } +// if ((++totalCount % 5000) == 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(insertQueryPrefix); +// pendingCount = 0; +// } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { if (br != null) { try { @@ -70,5 +92,7 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } + } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java index 4cdd33d..9867bff 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java @@ -5,16 +5,17 @@ import org.apache.log4j.Logger; import org.sitenv.vocabularies.loader.BaseCodeLoader; import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.List; +import javax.sql.DataSource; + /** * Created by Brian on 2/7/2016. */ @@ -22,8 +23,9 @@ public class SnomedLoader extends BaseCodeLoader implements VocabularyLoader { private static Logger logger = Logger.getLogger(SnomedLoader.class); - @Override - public void load(List filesToLoad, Connection connection) { + public long load(List filesToLoad, DataSource datasource) { + long n = 0; + JdbcTemplate t = new JdbcTemplate(datasource); BufferedReader br = null; FileReader fileReader = null; try { @@ -45,26 +47,49 @@ public void load(List filesToLoad, Connection connection) { String[] line = StringUtils.splitPreserveAllTokens(available, "\t", 9); String code = line[4]; String displayName = line[7]; +// if (pendingCount++ > 0) { +// insertQueryBuilder.append(","); +// } +// insertQueryBuilder.append("("); +// insertQueryBuilder.append("DEFAULT"); +// insertQueryBuilder.append(",'"); +// insertQueryBuilder.append(line[4].toUpperCase()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(line[7].toUpperCase().replaceAll("'", "''")); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(file.getParentFile().getName()); +// insertQueryBuilder.append("','"); +// insertQueryBuilder.append(CodeSystemOIDs.SNOMEDCT.codesystemOID()); +// insertQueryBuilder.append("')"); - buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, CodeSystemOIDs.SNOMEDCT.codesystemOID()); + n++; - if ((++totalCount % BATCH_SIZE) == 0) { - insertCode(insertQueryBuilder.toString(), connection); - insertQueryBuilder.clear(); - insertQueryBuilder.append(codeTableInsertSQLPrefix); - pendingCount = 0; - } + + buildCodeInsertQueryString(insertQueryBuilder, code.toUpperCase(), displayName.toUpperCase(), codeSystem, CodeSystemOIDs.SNOMEDCT.codesystemOID()); + t.update(insertQueryBuilder.toString()); + insertQueryBuilder.clear(); + insertQueryBuilder.append(codeTableInsertSQLPrefix); + + +// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),displayName.toUpperCase(),codeSystem,CodeSystemOIDs.SNOMEDCT.codesystemOID()); +// t.update(insertQueryPrefix,line[4].toUpperCase(),line[7].toUpperCase(),file.getParentFile().getName(),CodeSystemOIDs.SNOMEDCT.codesystemOID()); +// if ((++totalCount % 5000) == 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// insertQueryBuilder.clear(); +// insertQueryBuilder.append(insertQueryPrefix); +// pendingCount = 0; +// } } } } } - if (pendingCount > 0) { - insertCode(insertQueryBuilder.toString(), connection); - } +// if (pendingCount > 0) { +// doInsert(insertQueryBuilder.toString(), connection); +// } } catch (IOException e) { logger.error(e); - } catch (SQLException e) { - e.printStackTrace(); +// } catch (SQLException e) { +// e.printStackTrace(); } finally { if (br != null) { try { @@ -75,5 +100,7 @@ public void load(List filesToLoad, Connection connection) { } } } + return n; } + } diff --git a/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java b/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java index f7f716b..b9e5952 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java @@ -1,22 +1,26 @@ package org.sitenv.vocabularies.loader.valueset; -import com.monitorjbl.xlsx.StreamingReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import javax.sql.DataSource; + import org.apache.log4j.Logger; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; import org.sitenv.vocabularies.loader.BaseCodeLoader; import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; +//import com.monitorjbl.xlsx.StreamingReader; @Component(value = "VSAC") public class VsacLoader extends BaseCodeLoader implements VocabularyLoader { @@ -26,15 +30,18 @@ public class VsacLoader extends BaseCodeLoader implements VocabularyLoader { private static final int BATCH_SIZE = 1000; private static final String HEADER_ROW_FINDER_KEY = "CODE"; - public void load(List filesToLoad, Connection connection) { - String insertQueryPrefix = "insert into VALUESETS (ID, CODE, DISPLAYNAME, CODESYSTEMNAME, CODESYSTEMVERSION, CODESYSTEM, TTY, VALUESETNAME, VALUESETOID, VALUESETTYPE, VALUESETDEFINITIONVERSION, VALUESETSTEWARD) values (DEFAULT ,?,?,?,?,?,?,?,?,?,?,?)"; + public long load(List filesToLoad, DataSource datasource) { + String insertQueryPrefix = "insert into VALUESETS (CODE, DISPLAYNAME, CODESYSTEMNAME, CODESYSTEMVERSION, CODESYSTEM, TTY, VALUESETNAME, VALUESETOID, VALUESETTYPE, VALUESETDEFINITIONVERSION, VALUESETSTEWARD) values (?,?,?,?,?,?,?,?,?,?,?)"; + long n = 0; + JdbcTemplate templ = new JdbcTemplate(datasource); for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { try { logger.info("Loading Value Set File: " + file.getName()); - PreparedStatement preparedStatement = connection.prepareStatement(insertQueryPrefix); +// PreparedStatement preparedStatement = connection.prepareStatement(insertQueryPrefix); InputStream inputStream = new FileInputStream(file); - Workbook workBook = StreamingReader.builder().open(inputStream); +// Workbook workBook = StreamingReader.builder().open(inputStream); + Workbook workBook = WorkbookFactory.create(inputStream); for (int i = 1; i < workBook.getNumberOfSheets(); i++) { boolean headerRowFound = false; Sheet sheet = workBook.getSheetAt(i); @@ -48,63 +55,101 @@ public void load(List filesToLoad, Connection connection) { for(Row row : sheet){ if(row.getRowNum() < 6) { if (row.getRowNum() == 1) { + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); valueSetName = row.getCell(1).getStringCellValue().toUpperCase().trim(); } if (row.getRowNum() == 2) { + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); valueSetOid = row.getCell(1).getStringCellValue().toUpperCase().trim(); } if (row.getRowNum() == 3) { + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); valueSetType = row.getCell(1).getStringCellValue().toUpperCase().trim(); } if (row.getRowNum() == 4) { + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); valueSetVersion = row.getCell(1).getStringCellValue().toUpperCase().trim(); } if (row.getRowNum() == 5) { - valueSetSteward = row.getCell(1).getStringCellValue().toUpperCase().trim(); + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); + valueSetSteward = row.getCell(1).getStringCellValue().replaceAll("'", "''").toUpperCase().trim(); } } - if(headerRowFound && canProcessRow(row)){ - preparedStatement.setString(1, row.getCell(0).getStringCellValue().toUpperCase().trim()); - preparedStatement.setString(2, row.getCell(1).getStringCellValue().toUpperCase().trim()); - preparedStatement.setString(3, row.getCell(2).getStringCellValue().toUpperCase().trim()); - preparedStatement.setString(4, row.getCell(3).getStringCellValue().trim()); - preparedStatement.setString(5, row.getCell(4).getStringCellValue().toUpperCase().trim()); - preparedStatement.setString(6, row.getCell(5).getStringCellValue().toUpperCase().trim()); - preparedStatement.setString(7, valueSetName); - preparedStatement.setString(8, valueSetOid); - preparedStatement.setString(9, valueSetType); - preparedStatement.setString(10, valueSetVersion); - preparedStatement.setString(11, valueSetSteward); - preparedStatement.addBatch(); - valuesetDataRowCount++; - - if(valuesetDataRowCount % BATCH_SIZE == 0){ - preparedStatement.executeBatch(); - connection.commit(); - preparedStatement.clearBatch(); + if(row.getRowNum() > 10){ + if(row.getCell(0) != null) { + row.getCell(0).setCellType(Cell.CELL_TYPE_STRING); +// if (!StringUtils.isEmpty(row.getCell(0).getStringCellValue())) { + if(headerRowFound && canProcessRow(row)){ +// preparedStatement.setString(1, row.getCell(0).getStringCellValue().replaceAll("'", "''").toUpperCase().trim()); +// preparedStatement.setString(2, row.getCell(1).getStringCellValue().replaceAll("'", "''").toUpperCase().trim()); +// preparedStatement.setString(3, row.getCell(2).getStringCellValue().toUpperCase().trim()); +// preparedStatement.setString(4, row.getCell(3).getStringCellValue().trim()); +// preparedStatement.setString(5, row.getCell(4).getStringCellValue().toUpperCase().trim()); +// preparedStatement.setString(6, row.getCell(5).getStringCellValue().toUpperCase().trim()); +// preparedStatement.setString(7, valueSetName); +// preparedStatement.setString(8, valueSetOid); +// preparedStatement.setString(9, valueSetType); +// preparedStatement.setString(10, valueSetVersion); +// preparedStatement.setString(11, valueSetSteward); +// preparedStatement.addBatch(); + n++; + row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); + row.getCell(2).setCellType(Cell.CELL_TYPE_STRING); + row.getCell(3).setCellType(Cell.CELL_TYPE_STRING); + row.getCell(4).setCellType(Cell.CELL_TYPE_STRING); + row.getCell(5).setCellType(Cell.CELL_TYPE_STRING); + templ.update(insertQueryPrefix, row.getCell(0).getStringCellValue().toUpperCase().trim(), + row.getCell(1).getStringCellValue().toUpperCase().trim(), + row.getCell(2).getStringCellValue().toUpperCase().trim(), + row.getCell(3).getStringCellValue().trim(), + row.getCell(4).getStringCellValue().toUpperCase().trim(), + row.getCell(5).getStringCellValue().toUpperCase().trim(), + valueSetName, + valueSetOid, + valueSetType, + valueSetVersion, + valueSetSteward); + } } +// if(row.getRowNum() % 1000 == 0){ +// preparedStatement.executeBatch(); +// connection.commit(); +// preparedStatement.clearBatch(); +// valuesetDataRowCount++; + +// if(valuesetDataRowCount % BATCH_SIZE == 0){ +// preparedStatement.executeBatch(); +// connection.commit(); +// preparedStatement.clearBatch(); +// } +// } } if(!headerRowFound){ if(hasValueInCell(row, 0) && row.getCell(0).getStringCellValue().toUpperCase().trim().equals(HEADER_ROW_FINDER_KEY)){ headerRowFound = true; } +// preparedStatement.executeBatch(); +// connection.commit(); } - } - preparedStatement.executeBatch(); - connection.commit(); + } } workBook.close(); - } catch (IOException | SQLException e) { + } catch (InvalidFormatException e) { + logger.error("ERROR loading valueset. " + e.getLocalizedMessage()); + e.printStackTrace(); + } catch (IOException e) { logger.error("ERROR loading valueset. " + e.getLocalizedMessage()); e.printStackTrace(); } } } + return n; } + private boolean canProcessRow(Row row) { return hasCodevalueInFirstCell(row) && hasExpectedNumberOfCellsInRow(row); } diff --git a/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java b/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java index 9de9a44..5fcd8e2 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java +++ b/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java @@ -2,11 +2,14 @@ import org.sitenv.vocabularies.configuration.ConfiguredValidator; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.w3c.dom.Node; +import com.ximpleware.VTDNav; + import javax.xml.xpath.XPath; import java.util.List; public interface NodeValidation { - List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex); + List validateNode(ConfiguredValidator configuredValidator, XPath xpath, VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression); } diff --git a/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java new file mode 100644 index 0000000..5472953 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java @@ -0,0 +1,202 @@ +package org.sitenv.vocabularies.validation.dao; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.sql.DataSource; + +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.RowCountCallbackHandler; +import org.springframework.stereotype.Component; + +@Component +public class CodeSystemCodeDAO { + + private static Logger logger = Logger.getLogger(CodeSystemCodeDAO.class); + private static Map> CodeToCodeSystems = new HashMap>(); + private static Set CodeSystems = new HashSet(); + private static Map> DisplayNameToCodeSystems = new HashMap>(); + private static Map> CodeAndDisplayNameToCodeSystems = new HashMap>(); + private static Map> CodeAndDisplayNameAndCodeSystemOIDToCodeSystems = new HashMap>(); + private static Map> CodeSystemOIDToCodeSystems = new HashMap>(); + + private static boolean done = false; + + private void addRow(String code, String codeSystem, String displayName, String codeSystemOID) { + CodeSystems.add(codeSystem); + + if (CodeToCodeSystems.get(code) == null) { + CodeToCodeSystems.put(code, new HashSet()); + } + CodeToCodeSystems.get(code).add(codeSystem); + + if (DisplayNameToCodeSystems.get(displayName) == null) { + DisplayNameToCodeSystems.put(displayName, new HashSet()); + } + DisplayNameToCodeSystems.get(displayName).add(codeSystem); + + String key = "C/" + code + "/D/" + displayName; + if (CodeAndDisplayNameToCodeSystems.get(key) == null) { + CodeAndDisplayNameToCodeSystems.put(key, new HashSet()); + } + CodeAndDisplayNameToCodeSystems.get(key).add(codeSystem); + + key = "C/" + code + "/D/" + displayName + "/OID/" + codeSystemOID; + if (CodeAndDisplayNameAndCodeSystemOIDToCodeSystems.get(key) == null) { + CodeAndDisplayNameAndCodeSystemOIDToCodeSystems.put(key, new HashSet()); + } + CodeAndDisplayNameAndCodeSystemOIDToCodeSystems.get(key).add(codeSystem); + + if (CodeSystemOIDToCodeSystems.get(codeSystemOID) == null) { + CodeSystemOIDToCodeSystems.put(codeSystemOID, new HashSet()); + } + CodeSystemOIDToCodeSystems.get(codeSystemOID).add(codeSystem); + } + + public synchronized void loadCodes(DataSource ds) { + if (done) { + return; + } + // String sql = "SELECT codeSystem FROM Codes"; + // NamedParameterJdbcTemplate t = new + // NamedParameterJdbcTemplate(datasource); + // Map paramMap = new HashMap(); + // codes = t.queryForList(sql, paramMap, String.class); + + String sql = "SELECT code, displayName,codeSystem, codeSystemOID FROM Codes"; + JdbcTemplate tmpl = new JdbcTemplate(ds); + CodeSetRowHandler handler = new CodeSetRowHandler(); + tmpl.query(sql, handler); + + done = true; + logger.info("Code Maps initialized. Total rows:" + handler.n); + } + + public class CodeSetRowHandler implements RowCallbackHandler { + public long n = 0; + @Override + public void processRow(ResultSet arg0) throws SQLException { + String code = arg0.getString("code"); + String codeSystem = arg0.getString("codeSystem"); + String displayName = arg0.getString("displayName"); + String codeSystemOID = arg0.getString("codeSystemOID"); + addRow(code, codeSystem, displayName, codeSystemOID); + n++; + } + } + + // 1 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.codeSystem in (:codesystems)") + public boolean foundCodesystems(List codesystems) { + for (String s : codesystems) { + if (CodeSystems.contains(s)) { + return true; + } + } + return false; + } + + // 2 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.code = :code and c.displayName = :displayName and c.codeSystem + // in (:codesystems)") + public boolean foundCodeAndDisplayNameInCodesystem(String code, String displayName, List codesystems) { + String key = "C/" + code + "/D/" + displayName; + Set codeSystems = CodeAndDisplayNameToCodeSystems.get(key); + if (codeSystems == null) { + return false; + } + for (String s : codesystems) { + if (codeSystems.contains(s)) { + return true; + } + } + return false; + } + + // 3 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.code = :code and c.displayName = :displayName and + // c.codeSystemOID = :codeSystemOID and c.codeSystem in (:codesystems)") + // boolean + // foundCodeAndDisplayNameAndCodeSystemOIDInCodesystem(@Param("code")String + // code, @Param("displayName")String displayName, + // @Param("codeSystemOID")String codeSystemOID, + // @Param("codesystems")List codesystems); + public boolean foundCodeAndDisplayNameAndCodeSystemOIDInCodesystem(String code, String displayName, + String codeSystemOID, List codesystems) { + String key = "C/" + code + "/D/" + displayName + "/OID/" + codeSystemOID; + Set codeSystems = CodeAndDisplayNameAndCodeSystemOIDToCodeSystems.get(key); + if (codeSystems == null) { + return false; + } + for (String s : codesystems) { + if (codeSystems.contains(s)) { + return true; + } + } + return false; + } + + // 4 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.code = :code and c.codeSystem in (:codesystems)") + public boolean foundCodeInCodesystems(String code, List codesystems) { + Set codeSystems = CodeToCodeSystems.get(code); + if (codeSystems == null) { + return false; + } + + for (String s : codesystems) { + if (codeSystems.contains(s)) { + return true; + } + } + return false; + } + + // 5 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.displayName = :displayName and c.codeSystem in (:codesystems)") + public boolean foundDisplayNameInCodesystems(String displayName, List codesystems) { + Set codeSystems = DisplayNameToCodeSystems.get(displayName); + if (codeSystems == null) { + return false; + } + + for (String s : codesystems) { + if (codeSystems.contains(s)) { + return true; + } + } + return false; + } + + // 6 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM Code + // c WHERE c.codeSystemOID = :codeSystemOID and c.codeSystem in + // (:codesystems)") + // boolean foundCodeSystemOIDInCodesystems(@Param("codeSystemOID")String + // codeSystemOID, @Param("codesystems")List codesystems); + public boolean foundCodeSystemOIDInCodesystems(String codeSystemOID, List codesystems) { + Set codeSystems = CodeSystemOIDToCodeSystems.get(codeSystemOID); + if (codeSystems == null) { + return false; + } + + for (String s : codesystems) { + if (codeSystems.contains(s)) { + return true; + } + } + return false; + } +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java b/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java new file mode 100644 index 0000000..b0f3eb4 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java @@ -0,0 +1,271 @@ +package org.sitenv.vocabularies.validation.dao; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.sql.DataSource; + +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.stereotype.Component; + +@Component +public class ValueSetDAO { + + private static Logger logger = Logger.getLogger(ValueSetDAO.class); + private static Map> CodeToValueSetOID = new HashMap>(); + private static Set ValueSetOIDS = new HashSet(); + private static Map> CodeAndCodeSystemAndCodeSystemNameAndDisplayNameToValueSetOID = new HashMap>(); + private static Map> CodeSystemToValueSetOID = new HashMap>(); + private static Map> CodeAndCodeSystemToValueSetOID = new HashMap>(); + private static Map> DisplayNameAndCodeAndCodeSystemToValueSetOID = new HashMap>(); + private static Map> CodeSystemNameAndCodeAndCodeSystemToValueSetOID = new HashMap>(); + private static Map> CodeSystemNameToValueSetOID = new HashMap>(); + private static Map> DisplayNameToValueSetOID = new HashMap>(); + + private static boolean done = false; + + private void addRow(String code, String codeSystem, String displayName, String codeSystemName, + String valueSetOID) { + ValueSetOIDS.add(valueSetOID); + + Set codes = CodeToValueSetOID.get(code); + if (codes == null) { + CodeToValueSetOID.put(code, new HashSet()); + } + CodeToValueSetOID.get(code).add(valueSetOID); + + String key = "C/" + code + "/CS/" + codeSystem + "/CSN/" + codeSystemName + "/DN/" + displayName; + Set vsoids = CodeAndCodeSystemAndCodeSystemNameAndDisplayNameToValueSetOID.get(key); + if (vsoids == null) { + CodeAndCodeSystemAndCodeSystemNameAndDisplayNameToValueSetOID.put(key, new HashSet()); + } + CodeAndCodeSystemAndCodeSystemNameAndDisplayNameToValueSetOID.get(key).add(valueSetOID); + + vsoids = CodeSystemToValueSetOID.get(codeSystem); + if (vsoids == null) { + CodeSystemToValueSetOID.put(codeSystem, new HashSet()); + } + CodeSystemToValueSetOID.get(codeSystem).add(valueSetOID); + + key = "C/" + code + "/CS/" + codeSystem; + vsoids = CodeAndCodeSystemToValueSetOID.get(key); + if (vsoids == null) { + CodeAndCodeSystemToValueSetOID.put(key, new HashSet()); + } + CodeAndCodeSystemToValueSetOID.get(key).add(valueSetOID); + + key = "D/" + displayName + "/C/" + code + "/CS/" + codeSystem; + vsoids = DisplayNameAndCodeAndCodeSystemToValueSetOID.get(key); + if (vsoids == null) { + DisplayNameAndCodeAndCodeSystemToValueSetOID.put(key, new HashSet()); + } + DisplayNameAndCodeAndCodeSystemToValueSetOID.get(key).add(valueSetOID); + + key = "CSN/" + codeSystemName + "/C/" + code + "/CS/" + codeSystem; + vsoids = CodeSystemNameAndCodeAndCodeSystemToValueSetOID.get(key); + if (vsoids == null) { + CodeSystemNameAndCodeAndCodeSystemToValueSetOID.put(key, new HashSet()); + } + CodeSystemNameAndCodeAndCodeSystemToValueSetOID.get(key).add(valueSetOID); + + if (CodeSystemNameToValueSetOID.get(codeSystemName)==null) { + CodeSystemNameToValueSetOID.put(codeSystemName, new HashSet()); + } + CodeSystemNameToValueSetOID.get(codeSystemName).add(valueSetOID); + + if (DisplayNameToValueSetOID.get(displayName)==null) { + DisplayNameToValueSetOID.put(displayName, new HashSet()); + } + DisplayNameToValueSetOID.get(displayName).add(valueSetOID); + } + + public synchronized void loadValueSets(DataSource ds) { + if (done) { + return; + } + + String sql = "SELECT code, displayName,codeSystem, codeSystemName, valuesetOid FROM ValueSets"; + JdbcTemplate tmpl = new JdbcTemplate(ds); + ValueSetRowHandler handler = new ValueSetRowHandler(); + tmpl.query(sql, handler); + done = true; + logger.info("ValueSet Maps initialized. Total rows:" + handler.n); + logger.info("ValueSetOIDs:" + ValueSetOIDS.size()); + } + + public class ValueSetRowHandler implements RowCallbackHandler { + public long n = 0; + @Override + public void processRow(ResultSet arg0) throws SQLException { + String code = arg0.getString("code"); + String codeSystem = arg0.getString("codeSystem"); + String displayName = arg0.getString("displayName"); + String codeSystemName = arg0.getString("codeSystemName"); + String valueSetOID = arg0.getString("valuesetOid"); + addRow(code, codeSystem, displayName, codeSystemName, valueSetOID); + n++; + } + } + + // 1 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.code = :code and c.codeSystem = :codeSystem and + // c.codeSystemName = :codeSystemName and c.displayName = :displayName and + // c.valuesetOid in (:valuesetOids)") + public boolean existsByCodeAndCodeSystemAndCodeSystemNameAndDisplayNameInValuesetOid(String code, String codeSystem, + String codeSystemName, String displayName, List valuesetOids) { + + String key = "C/" + code + "/CS/" + codeSystem + "/CSN/" + codeSystemName + "/DN/" + displayName; + Set oids = CodeAndCodeSystemAndCodeSystemNameAndDisplayNameToValueSetOID.get(key); + if (oids == null) { + return false; + } + for (String oid : valuesetOids) { + if (oids.contains(oid)) { + return true; + } + } + return false; + } + + // 2 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.valuesetOid in (:valuesetOids)") + public boolean valuesetOidsExists(List valuesetOids) { + for (String vsoid : valuesetOids) { + if (ValueSetOIDS.contains(vsoid)) { + return true; + } + } + return false; + } + + // 3 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.codeSystem = :codeSystem and c.valuesetOid in + // (:valuesetOids)") + public boolean codeSystemExistsInValueset(String codeSystem, List valuesetOids) { + Set oids = CodeSystemToValueSetOID.get(codeSystem); + if (oids == null) { + return false; + } + for (String oid : valuesetOids) { + if (oids.contains(oid)) { + return true; + } + } + return false; + } + + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.code = :code and c.codeSystem = :codeSystem and + // c.valuesetOid in (:valuesetOids)") + public boolean codeExistsByCodeSystemInValuesetOid(String code, String codeSystem, List valuesetOids) { + + String key = "C/" + code + "/CS/" + codeSystem; + Set oids = CodeAndCodeSystemToValueSetOID.get(key); + if (oids == null) { + return false; + } + for (String oid : valuesetOids) { + if (oids.contains(oid)) { + return true; + } + } + return false; + } + + // 4 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.displayName = :displayName and c.code = :code and + // c.codeSystem = :codeSystem and c.valuesetOid in (:valuesetOids)") + public boolean displayNameExistsForCodeByCodeSystemInValueset(String displayName, String code, String codeSystem, + List valuesetOids) { + String key = "D/" + displayName + "/C/" + code + "/CS/" + codeSystem; + Set oids = DisplayNameAndCodeAndCodeSystemToValueSetOID.get(key); + if (oids == null) { + return false; + } + for (String oid : valuesetOids) { + if (oids.contains(oid)) { + return true; + } + } + return false; + } + + // 5 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.codeSystemName = :codeSystemName and c.code = + // :code and c.codeSystem = :codeSystem and c.valuesetOid in + // (:valuesetOids)") + public boolean codeSystemNameExistsForCodeByCodeSystemInValueset(String codeSystemName, String code, + String codeSystem, List valuesetOids) { + String key = "CSN/" + codeSystemName + "/C/" + code + "/CS/" + codeSystem; + Set oids = CodeSystemNameAndCodeAndCodeSystemToValueSetOID.get(key); + if (oids == null) { + return false; + } + for (String oid : valuesetOids) { + if (oids.contains(oid)) { + return true; + } + } + return false; + } + + // 6 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM VsacValueSet c WHERE c.displayName = :displayName and c.valuesetOid in (:valuesetOids)") + public boolean displayNameExistsInValueset(String displayName, List valueSetOids) { + Set codes = DisplayNameToValueSetOID.get(displayName); + if (codes == null) { + return false; + } + for (String vsoid : valueSetOids) { + if (codes.contains(vsoid)) { + return true; + } + } + return false; + } + + // 7 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM VsacValueSet c WHERE c.codeSystemName = :codeSystemName and c.valuesetOid in (:valuesetOids)") + public boolean codeSystemNameExistsInValueset(String codeSystemName, + List valuesetOids) { + Set codes = CodeSystemNameToValueSetOID.get(codeSystemName); + if (codes == null) { + return false; + } + for (String vsoid : valuesetOids) { + if (codes.contains(vsoid)) { + return true; + } + } + return false; + } + + // 8 + // @Query("SELECT CASE WHEN COUNT(c) > 0 THEN true ELSE false END FROM + // VsacValueSet c WHERE c.code = :code and c.valuesetOid in + // (:valuesetOids)") + public boolean codeExistsInValueset(String code, List valuesetOids) { + Set codes = CodeToValueSetOID.get(code); + if (codes == null) { + return false; + } + for (String vsoid : valuesetOids) { + if (codes.contains(vsoid)) { + return true; + } + } + return false; + } +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/dto/NodeValidationResult.java b/src/main/java/org/sitenv/vocabularies/validation/dto/NodeValidationResult.java index a4b3b4e..702cf2f 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/dto/NodeValidationResult.java +++ b/src/main/java/org/sitenv/vocabularies/validation/dto/NodeValidationResult.java @@ -6,7 +6,16 @@ public class NodeValidationResult { private int baseNodeIndex; private String configuredXpathExpression; private int nodeIndex; + public long getErrorOffset() { + return errorOffset; + } + + public void setErrorOffset(long errorOffset) { + this.errorOffset = errorOffset; + } + private long errorOffset = 0; + private String requestedCode; private String requestedCodeSystem; private String requestedCodeSystemName; @@ -16,6 +25,7 @@ public class NodeValidationResult { private String requestedText; private String configuredAllowableValuesetOidsForNode; private String configuredAllowableCodesystemNamesForNode; + private String ruleID; private boolean nodeCodeSystemOIDFound; private boolean nodeValuesetsFound; @@ -183,4 +193,12 @@ public boolean isNodeCodeSystemOIDFound() { public void setNodeCodeSystemOIDFound(boolean nodeCodeSystemOIDFound) { this.nodeCodeSystemOIDFound = nodeCodeSystemOIDFound; } + + public String getRuleID() { + return this.ruleID; + } + + public void setRuleID(String id) { + this.ruleID = id; + } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotObjectPoolInitializer.java b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotObjectPoolInitializer.java new file mode 100644 index 0000000..12a6389 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotObjectPoolInitializer.java @@ -0,0 +1,31 @@ +package org.sitenv.vocabularies.validation.pool; + +import java.util.List; + +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.configuration.ConfiguredExpression; + +public class AutoPilotObjectPoolInitializer { + + private static Logger logger = Logger.getLogger(AutoPilotObjectPoolInitializer.class); + private static boolean done = false; + + private void initPools(List ces) throws Exception { + if (done) { + return; + } + long s = System.currentTimeMillis(); + for (ConfiguredExpression ce : ces) { + AutoPilotWrapper apw = AutoPilotPool.newInstance().borrow(ce.getConfiguredXpathExpression()); + AutoPilotPool.newInstance().returnObject(apw); + + } + logger.info("Init AutoPilot ObjectPools in " + (System.currentTimeMillis() - s) + " ms for " + ces.size() + " expressions."); + done = true; + } + + public void initFromExpressions(List expressions) throws Exception { + initPools(expressions); + } + +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotPool.java b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotPool.java new file mode 100644 index 0000000..af65b54 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotPool.java @@ -0,0 +1,43 @@ +package org.sitenv.vocabularies.validation.pool; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; + +public class AutoPilotPool { + + private static Map POOLS = new HashMap(); + private static AutoPilotPool pool = null; + + public static AutoPilotPool newInstance() { + if (pool==null) { + pool = new AutoPilotPool(); + } + return pool; + } + + public AutoPilotWrapper borrow(String xpath) throws Exception { + AutoPilotWrapperPool apwpool = POOLS.get(xpath); + if (apwpool == null) { + + GenericObjectPoolConfig config = new GenericObjectPoolConfig(); + config.setMaxIdle(200); + config.setMaxTotal(200); + config.setMinIdle(100); + config.setTestOnBorrow(true); + config.setTestOnReturn(true); + + apwpool = new AutoPilotWrapperPool(new AutoPilotWrapperFactory(xpath), config); + POOLS.put(xpath, apwpool); + } + return apwpool.borrowObject(); + } + + public void returnObject(AutoPilotWrapper apw) { + AutoPilotWrapperPool apwpool = POOLS.get(apw.getXPath()); + if (apwpool!=null) { + apwpool.returnObject(apw); + } + } +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapper.java b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapper.java new file mode 100644 index 0000000..3b29276 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapper.java @@ -0,0 +1,62 @@ +package org.sitenv.vocabularies.validation.pool; + +import java.util.UUID; + +import com.ximpleware.AutoPilot; + +public class AutoPilotWrapper { + + private AutoPilot autopilot; + private String xpath; + private boolean ok = false; + + private final UUID objID = UUID.randomUUID(); + + /* + * Have to implement the equals method, since Spring puts a proxy around the object. + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AutoPilotWrapper)) { + return false; + } + AutoPilotWrapper w = (AutoPilotWrapper) o; + return (objID == null) ? w.objID==null : objID.equals(w.objID); + } + + public AutoPilotWrapper(String xpath) { + try { + this.xpath = xpath; + this.autopilot = new AutoPilot(); + this.autopilot.declareXPathNameSpace("sdtc", "urn:hl7-org:sdtc"); + this.autopilot.declareXPathNameSpace("v3", "urn:hl7-org:v3"); + this.autopilot.declareXPathNameSpace("voc", "urn:hl7-org:v3/voc"); + this.autopilot.declareXPathNameSpace("", "urn:hl7-org:v3"); + this.autopilot.selectXPath(this.xpath); + ok = true; + } catch (Exception e) { + e.printStackTrace(); + } + } + + public AutoPilot getAutoPilot() { + this.autopilot.resetXPath(); + return this.autopilot; + } + + public String getXPath() { + return xpath; + } + + public boolean isValid() { + return ok; + } + + public void reset() { + // void + } +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperFactory.java b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperFactory.java new file mode 100644 index 0000000..5d17d4b --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperFactory.java @@ -0,0 +1,36 @@ +package org.sitenv.vocabularies.validation.pool; + +import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.impl.DefaultPooledObject; + +public class AutoPilotWrapperFactory extends BasePooledObjectFactory { + + private String xpath; + + public AutoPilotWrapperFactory(String xpath) { + super(); + this.xpath=xpath; + } + + @Override + public AutoPilotWrapper create() throws Exception { + return new AutoPilotWrapper(xpath); + } + + @Override + public PooledObject wrap(AutoPilotWrapper parser) { + return new DefaultPooledObject(parser); + } + + @Override + public void passivateObject(PooledObject po) throws Exception { + po.getObject().reset(); + } + + @Override + public boolean validateObject(PooledObject po) { + return po.getObject().isValid(); + } + +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperPool.java b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperPool.java new file mode 100644 index 0000000..e413f7e --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/pool/AutoPilotWrapperPool.java @@ -0,0 +1,17 @@ +package org.sitenv.vocabularies.validation.pool; + +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; + +public class AutoPilotWrapperPool extends GenericObjectPool { + + public AutoPilotWrapperPool(PooledObjectFactory factory) { + super(factory); + } + + public AutoPilotWrapperPool(PooledObjectFactory factory, GenericObjectPoolConfig config) { + super(factory, config); + } + +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java new file mode 100644 index 0000000..ad7c603 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java @@ -0,0 +1,40 @@ +package org.sitenv.vocabularies.validation.services; + +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.configuration.ConfiguredExpression; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; +import org.w3c.dom.Document; + +import com.ximpleware.VTDNav; + +public class ValidateRequest { + + private ConfiguredExpression ce; + private ConfiguredExpressionFilter filter; + private Logger logger; + private VTDNav nv; + + public Logger getLogger() { + return this.logger; + } + + public ValidateRequest(ConfiguredExpression ce, ConfiguredExpressionFilter filter, Logger lgr, VTDNav vn) { + this.ce = ce; + this.filter = filter; + this.logger = lgr; + this.nv = vn; + } + + public VTDNav getNav() { + return this.nv; + } + + public ConfiguredExpressionFilter getFilter() { + return this.filter; + } + + public ConfiguredExpression getConfiguredExpression() { + return this.ce; + } + +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java new file mode 100644 index 0000000..b3f71a1 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java @@ -0,0 +1,127 @@ +package org.sitenv.vocabularies.validation.services; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; + +import javax.annotation.Resource; +import javax.xml.namespace.NamespaceContext; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +import org.sitenv.vocabularies.configuration.ConfiguredExpression; +import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.NodeValidation; +import org.sitenv.vocabularies.validation.NodeValidatorFactory; +import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; +import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; +import org.sitenv.vocabularies.validation.pool.AutoPilotPool; +import org.sitenv.vocabularies.validation.pool.AutoPilotWrapper; +import org.sitenv.vocabularies.validation.utils.CCDADocumentNamespaces; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.AsyncResult; +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; + +import com.ximpleware.AutoPilot; +import com.ximpleware.VTDNav; + +@Component +public class ValidateWorker { + + private XPathFactory xPathFactory; + + public ValidateWorker() { + this.xPathFactory = XPathFactory.newInstance(); + } + + @Resource(name = "vocabularyValidationConfigurations") + List vocabularyValidationConfigurations; + + @Resource(name = "vocabularyValidationConfigurationsForMu2") + List vocabularyValidationConfigurationsForMu2; + + @Autowired + NodeValidatorFactory vocabularyValidatorFactory; + + @Async("taskExecutor") + public Future>> doWork(ValidateRequest req) { + + Map> vocabularyValidationResultMap = getInitializedResultMap(); + + ConfiguredExpression configuredExpression = req.getConfiguredExpression(); + XPath xpath = null; + String configuredXpathExpression = configuredExpression.getConfiguredXpathExpression(); + AutoPilotWrapper apw = null; + try { + apw = AutoPilotPool.newInstance().borrow(configuredXpathExpression); + + VTDNav vn = req.getNav().cloneNav(); + AutoPilot ap = apw.getAutoPilot(); + ap.bind(vn); + + int i = 0; + while (ap.evalXPath() != -1) { + i++; + List vocabularyValidationResults = new ArrayList<>(); + boolean validNode = false; + Iterator configIterator = configuredExpression.getConfiguredValidators() + .iterator(); + while (configIterator.hasNext() && !validNode) { + ConfiguredValidator configuredValidator = configIterator.next(); + NodeValidation vocabularyValidator = vocabularyValidatorFactory + .getVocabularyValidator(configuredValidator.getName()); + List tempResults = vocabularyValidator.validateNode(configuredValidator, + xpath, vn, i, req.getFilter(), configuredXpathExpression); + if (foundValidationError(tempResults)) { + vocabularyValidationResults.addAll(tempResults); + } else { + vocabularyValidationResults.clear(); + vocabularyValidationResults.addAll(tempResults); + validNode = true; + } + } + + for (VocabularyValidationResult vocabularyValidationResult : vocabularyValidationResults) { + vocabularyValidationResult.getNodeValidationResult() + .setConfiguredXpathExpression(configuredXpathExpression); + vocabularyValidationResultMap + .get(vocabularyValidationResult.getVocabularyValidationResultLevel().getResultType()) + .add(vocabularyValidationResult); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (apw != null) { + AutoPilotPool.newInstance().returnObject(apw); + } + } + + return new AsyncResult>>(vocabularyValidationResultMap); + } + + private Map> getInitializedResultMap() { + Map> resultMap = new HashMap<>(); + resultMap.put("errors", new ArrayList()); + resultMap.put("warnings", new ArrayList()); + resultMap.put("info", new ArrayList()); + return resultMap; + } + + private boolean foundValidationError(List results) { + for (VocabularyValidationResult result : results) { + if (result.getVocabularyValidationResultLevel().equals(VocabularyValidationResultLevel.SHALL)) { + return true; + } + } + return false; + } +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java b/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java index 41d0f46..85f7b20 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java +++ b/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java @@ -1,5 +1,32 @@ package org.sitenv.vocabularies.validation.services; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; + +import javax.annotation.Resource; +import javax.xml.namespace.NamespaceContext; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredExpression; import org.sitenv.vocabularies.configuration.ConfiguredValidator; import org.sitenv.vocabularies.validation.NodeValidation; @@ -7,6 +34,7 @@ import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; import org.sitenv.vocabularies.validation.utils.CCDADocumentNamespaces; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionLevelFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; @@ -14,137 +42,224 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; -import javax.annotation.Resource; -import javax.xml.namespace.NamespaceContext; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.IOException; -import java.io.InputStream; -import java.util.*; +import com.ximpleware.VTDGen; +import com.ximpleware.VTDNav; /** * Created by Brian on 2/10/2016. */ @Service public class VocabularyValidationService { - @Resource(name="vocabularyValidationConfigurations") - List vocabularyValidationConfigurations; - @Resource(name="documentBuilder") - DocumentBuilder documentBuilder; - @Resource(name="xPathFactory") - XPathFactory xPathFactory; - @Autowired - NodeValidatorFactory vocabularyValidatorFactory; - - public List validate(String uri) throws IOException, SAXException { - Document doc = documentBuilder.parse(uri); - return this.validate(doc); - } - - public List validate(InputStream stream) throws IOException, SAXException { - Document doc = documentBuilder.parse(stream); - return this.validate(doc); - } - - public List validate(Document doc) { - Map> vocabularyValidationResultMap = getInitializedResultMap(); - if (doc != null) { - String configuredXpathExpression = ""; - try { - XPath xpath = getNewXpath(doc); - for (ConfiguredExpression configuredExpression : vocabularyValidationConfigurations) { - configuredXpathExpression = configuredExpression.getConfiguredXpathExpression(); - NodeList nodes = findAllDocumentNodesByXpathExpression(xpath, configuredXpathExpression, doc); - for (int i = 0; i < nodes.getLength(); i++) { - Node node = nodes.item(i); - List vocabularyValidationResults = new ArrayList<>(); - boolean validNode = false; - Iterator configIterator = configuredExpression.getConfiguredValidators().iterator(); - while(configIterator.hasNext() && !validNode){ - ConfiguredValidator configuredValidator = (ConfiguredValidator) configIterator.next(); - NodeValidation vocabularyValidator = vocabularyValidatorFactory.getVocabularyValidator(configuredValidator.getName()); - List tempResults = vocabularyValidator.validateNode(configuredValidator, xpath, node, i); - if(foundValidationError(tempResults)) { - vocabularyValidationResults.addAll(tempResults); - }else { - vocabularyValidationResults.clear(); - vocabularyValidationResults.addAll(tempResults); - validNode = true; - } - } - - for (VocabularyValidationResult vocabularyValidationResult : vocabularyValidationResults) { - vocabularyValidationResult.getNodeValidationResult().setConfiguredXpathExpression(configuredXpathExpression); - vocabularyValidationResultMap.get(vocabularyValidationResult.getVocabularyValidationResultLevel().getResultType()).add(vocabularyValidationResult); - } - - } - - } - } catch (XPathExpressionException e) { - System.err.println("ERROR VALIDATING DOCUMENT FOR THE FOLLOWING CONFIGURED EXPRESSION: " + configuredXpathExpression); - } - } - return convertMapToList(vocabularyValidationResultMap); - } - - private Map> getInitializedResultMap() { - Map> resultMap = new HashMap<>(); - resultMap.put("errors", new ArrayList()); - resultMap.put("warnings", new ArrayList()); - resultMap.put("info", new ArrayList()); - return resultMap; - } - - private XPath getNewXpath(final Document doc){ - XPath xpath = xPathFactory.newXPath(); - xpath.setNamespaceContext(new NamespaceContext() { - @Override - public String getNamespaceURI(String prefix) { - String nameSpace; - if(CCDADocumentNamespaces.sdtc.name().equals(prefix)){ - nameSpace = CCDADocumentNamespaces.sdtc.getNamespace(); - }else { - nameSpace = CCDADocumentNamespaces.defaultNameSpaceForCcda.getNamespace(); - } - return nameSpace; - } - - @Override - public String getPrefix(String namespaceURI) { - return null; - } - - @Override - public Iterator getPrefixes(String namespaceURI) { - return null; - } - }); - return xpath; - } - - private static NodeList findAllDocumentNodesByXpathExpression(XPath xpath, String configuredXpath, Document doc) throws XPathExpressionException { - NodeList result = (NodeList) xpath.compile(configuredXpath).evaluate(doc, XPathConstants.NODESET); - return result; - } - - private boolean foundValidationError(List results){ - for(VocabularyValidationResult result : results){ - if(result.getVocabularyValidationResultLevel().equals(VocabularyValidationResultLevel.SHALL)){ - return true; - } - } - return false; - } - - private List convertMapToList(Map> resultMap) { - List results = new ArrayList<>(); - for(ArrayList resultList : resultMap.values()){ - results.addAll(resultList); - } - return results; - } + + private static final Logger logger = Logger.getLogger(VocabularyValidationService.class); + + @Resource(name = "vocabularyValidationConfigurations") + List vocabularyValidationConfigurations; + + /* + * Following resource is defined to handle MU2 document validation to + * support CCDA R1.1 document templates This variable holds only MU2 + * specific validation expressions + */ + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + @Resource(name = "vocabularyValidationConfigurationsForMu2") + List vocabularyValidationConfigurationsForMu2; + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + // @Resource(name="documentBuilder") + // DocumentBuilder documentBuilder; + @Resource(name = "documentBuilderFactory") + DocumentBuilderFactory documentBuilderFactory; + + private DocumentBuilder getDocumentBuilder() throws ParserConfigurationException { + return documentBuilderFactory.newDocumentBuilder(); + } + + @Autowired + NodeValidatorFactory vocabularyValidatorFactory; + + @Autowired + ValidateWorker worker; + + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + @Resource(name = "xPathFactory") + XPathFactory xPathFactory; + + // @Autowired + // VocabularyValidatorFactory vocabularyValidatorFactory; + + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + public List validate(String uri, String severityLevel) + throws IOException, SAXException, ParserConfigurationException { + // Document doc = documentBuilder.parse(uri); + Document doc = getDocumentBuilder().parse(uri); + return this.validate(doc, uri, severityLevel); + } + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + // ------------------------- INTERNAL CODE CHANGE START + // -------------------------- + public List validate(InputStream stream, String severityLevel) + throws IOException, SAXException, ParserConfigurationException { + // Document doc = documentBuilder.parse(stream); + String xml = IOUtils.toString(stream); + stream.reset(); + Document doc = getDocumentBuilder().parse(stream); + return this.validate(doc, xml, severityLevel); + } + // ------------------------- INTERNAL CODE CHANGE END + // -------------------------- + + public List validate(Document doc, String xmlDoc, String severityLevel) { + Map> vocabularyValidationResultMap = getInitializedResultMap(); + if (doc != null) { + String configuredXpathExpression = ""; + try { + XPath xpath = getNewXpath(doc); + + /* + * Updated for loop to call getVocabValidationConfigurations + * method to validate respective document template specific + * expressions. getVocabValidationConfigurations() method + * returns the expression list based on the document templates + * (CCDA R1.1, CCDA R2.1) + * + */ + // ------------------------- INTERNAL CODE CHANGE START -------------------------- + // for (ConfiguredExpression configuredExpression : + // vocabularyValidationConfigurations) { + + List docValidations = getVocabValidationConfigurations(doc, xpath); + List filteredList = new ArrayList(docValidations.size()); + ConfiguredExpressionLevelFilter filter = new ConfiguredExpressionLevelFilter(severityLevel); + for (ConfiguredExpression configuredExpression : docValidations) { + ConfiguredExpression e = filter.accept(configuredExpression); + if (e != null) { + filteredList.add(e); + } + } + + logger.info("Document qualifying expressions:" + docValidations.size() + " filtered to level(" + severityLevel + ") leaves:" + filteredList.size()); + + try { + VTDGen vg = new VTDGen(); + vg.setDoc(xmlDoc.getBytes("UTF-8")); + vg.parse(true); + VTDNav nav = vg.getNav(); + + List>>> futures = new ArrayList>>>(); + for (ConfiguredExpression configuredExpression : filteredList) { + ValidateRequest r = new ValidateRequest(configuredExpression, filter, logger, nav); + futures.add(worker.doWork(r)); + } + for (Future>> future : futures) { + try { + Map> r = future.get(); + for (String key : r.keySet()) { + vocabularyValidationResultMap.get(key).addAll(r.get(key)); + } + } catch (Exception e) { + logger.error("Error gettting future result.",e); + e.printStackTrace(); + } + } + } catch (Exception e) { + logger.error("Error submitting futures.",e); + e.printStackTrace(); + } + } catch (Exception e) { + logger.error("Error validating document for the following configured expression:" + configuredXpathExpression,e); + System.err.println("ERROR VALIDATING DOCUMENT FOR THE FOLLOWING CONFIGURED EXPRESSION: " + + configuredXpathExpression); + e.printStackTrace(); + } + + } + return convertMapToList(vocabularyValidationResultMap); + } + + private Map> getInitializedResultMap() { + Map> resultMap = new HashMap<>(); + resultMap.put("errors", new ArrayList()); + resultMap.put("warnings", new ArrayList()); + resultMap.put("info", new ArrayList()); + return resultMap; + } + + private XPath getNewXpath(final Document doc) { + XPath xpath = xPathFactory.newXPath(); + xpath.setNamespaceContext(new NamespaceContext() { + @Override + public String getNamespaceURI(String prefix) { + String nameSpace; + if (CCDADocumentNamespaces.sdtc.name().equals(prefix)) { + nameSpace = CCDADocumentNamespaces.sdtc.getNamespace(); + } else { + nameSpace = CCDADocumentNamespaces.defaultNameSpaceForCcda.getNamespace(); + } + return nameSpace; + } + + @Override + public String getPrefix(String namespaceURI) { + return null; + } + + @Override + public Iterator getPrefixes(String namespaceURI) { + return null; + } + }); + return xpath; + } + + + private List convertMapToList( + Map> resultMap) { + List results = new ArrayList<>(); + for (ArrayList resultList : resultMap.values()) { + results.addAll(resultList); + } + return results; + } + + /* + * Following method is added to differentiate weather CCDA document template + * is R 1.1 or R2.1 Load the xpath configuration list based on the document + * version (MU2 , MU3) + */ + // ------------------------- INTERNAL CODE CHANGE START -------------------------- + + private List getVocabValidationConfigurations(Document doc, XPath xpath) + throws XPathExpressionException { + + Number num = (Number) xpath + .compile("count(/v3:ClinicalDocument/v3:templateId[@root='2.16.840.1.113883.10.20.22.1.1']/@extension)") + .evaluate(doc, XPathConstants.NUMBER); + + List ret = null; + if (num.intValue() == 0) {// MU2 CCDA R1.1 document + ret = vocabularyValidationConfigurationsForMu2; + if (ret.size() == 0) { + logger.warn("Validating an MU2 CCDA R1.1 document, but found no configured vocabularyValidations!!!"); + } + } else { // MU3 CCDA R2.1 document + ret = vocabularyValidationConfigurations; + if (ret.size() == 0) { + logger.warn("Validating an MU3 CCDA R2.1 document, but found no configured vocabularyValidations!!!"); + } + } + return ret; + } + + // ------------------------- INTERNAL CODE CHANGE END -------------------------- + } diff --git a/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionFilter.java b/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionFilter.java new file mode 100644 index 0000000..d478672 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionFilter.java @@ -0,0 +1,10 @@ +package org.sitenv.vocabularies.validation.utils; + +import org.sitenv.vocabularies.configuration.ConfiguredExpression; +import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; + +public interface ConfiguredExpressionFilter { + + ConfiguredExpression accept(ConfiguredExpression ce); + boolean isEnabled(VocabularyValidationResultLevel lvl); +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionLevelFilter.java b/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionLevelFilter.java new file mode 100644 index 0000000..194dd16 --- /dev/null +++ b/src/main/java/org/sitenv/vocabularies/validation/utils/ConfiguredExpressionLevelFilter.java @@ -0,0 +1,130 @@ +package org.sitenv.vocabularies.validation.utils; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.sitenv.vocabularies.configuration.ConfiguredExpression; +import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; +import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; + +public class ConfiguredExpressionLevelFilter implements ConfiguredExpressionFilter { + + private String level = "info"; + private int iLevel = -1; + + private int INFO_LEVEL = 1; + private int WARN_LEVEL = 2; + private int ERROR_LEVEL = 3; + + public ConfiguredExpressionLevelFilter(String level) { + if (level != null) { + this.level = level.toLowerCase(); + } + if (this.level.contains("info")) { + iLevel = INFO_LEVEL; + } else if (this.level.contains("warning")) { + iLevel = WARN_LEVEL; + } else if (this.level.contains("error")) { + iLevel = ERROR_LEVEL; + } + } + + public boolean isEnabled(VocabularyValidationResultLevel lvl) { + if (lvl == null) { + return true; + } + // SHALL is always enabled. + if (lvl == VocabularyValidationResultLevel.SHALL) { + return true; + } + if (lvl == VocabularyValidationResultLevel.SHOULD) { + return iLevel <= WARN_LEVEL; + } + if (lvl == VocabularyValidationResultLevel.MAY) { + return iLevel <= INFO_LEVEL; + } + // SHALL is always enabled. + return true; + } + + @Override + public ConfiguredExpression accept(ConfiguredExpression ce) { + + // If we're asking for INFO level, all expressions qualify, no need to + // check. + if (iLevel == INFO_LEVEL) { + return ce; + } + + // MAY ("info"), + // SHOULD ("warnings"), + // SHALL ("errors"); + + boolean anyValidatorQualifies = false; + Iterator i = ce.getConfiguredValidators().iterator(); + while (i.hasNext() && !anyValidatorQualifies) { + ConfiguredValidator v = i.next(); + int ruleLevel = INFO_LEVEL; + ConfiguredValidationResultSeverityLevel vlvl = v.getConfiguredValidationResultSeverityLevel(); + if (vlvl == null) { + ruleLevel = ERROR_LEVEL; + } else { + + if (vlvl.getCodeSeverityLevel().equals(VocabularyValidationResultLevel.MAY.name())) { + ruleLevel = INFO_LEVEL; + } + if (vlvl.getCodeSeverityLevel().equals(VocabularyValidationResultLevel.SHOULD.name())) { + ruleLevel = WARN_LEVEL; + } + if (vlvl.getCodeSeverityLevel().equals(VocabularyValidationResultLevel.SHALL.name())) { + // We always need to check SHALL rules. + ruleLevel = ERROR_LEVEL; + } + } + anyValidatorQualifies = (ruleLevel >= iLevel); + } + + if (!anyValidatorQualifies) { + return null; + } + + // Need to make a deep copy of the original expression with only the + // qualifying validator(s) + ConfiguredExpression f = new ConfiguredExpression(); + f.setConfiguredXpathExpression(ce.getConfiguredXpathExpression()); + List validators = new ArrayList(ce.getConfiguredValidators().size()); + f.setConfiguredValidators(validators); + for (ConfiguredValidator configuredValidator : ce.getConfiguredValidators()) { + String codeSeverityLevel = VocabularyValidationResultLevel.SHALL.name(); + ConfiguredValidationResultSeverityLevel lvl = configuredValidator + .getConfiguredValidationResultSeverityLevel(); + if (lvl != null) { + codeSeverityLevel = lvl.getCodeSeverityLevel(); + } + if (codeSeverityLevel.equals(VocabularyValidationResultLevel.MAY.name())) { + if (iLevel <= INFO_LEVEL) { + validators.add(configuredValidator); + } + } + if (codeSeverityLevel.equals(VocabularyValidationResultLevel.SHOULD.name())) { + if (iLevel <= WARN_LEVEL) { + validators.add(configuredValidator); + } + } + if (codeSeverityLevel.equals(VocabularyValidationResultLevel.SHALL.name())) { + if (iLevel <= ERROR_LEVEL) { + validators.add(configuredValidator); + } + } + } + + if (validators.size() == 0) { + return null; + } + + return f; + } + +} diff --git a/src/main/java/org/sitenv/vocabularies/validation/utils/XpathUtils.java b/src/main/java/org/sitenv/vocabularies/validation/utils/XpathUtils.java index 70bfffe..16a8b6a 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/utils/XpathUtils.java +++ b/src/main/java/org/sitenv/vocabularies/validation/utils/XpathUtils.java @@ -3,35 +3,72 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; +import com.ximpleware.VTDNav; + /** * Created by Brian on 10/20/2015. */ public class XpathUtils { - public static String buildXpathFromNode(Node node) { - return getXPath(node, ""); - } - - private static String getXPath(Node node, String xpath) { - String elementName = ""; - if (node instanceof Element) { - elementName = node.getNodeName(); - int prev_siblings = 1; - Node prev_sibling = node.getPreviousSibling(); - while (null != prev_sibling) { - if (prev_sibling.getNodeType() == node.getNodeType()) { - if (prev_sibling.getNodeName().equalsIgnoreCase( - node.getNodeName())) { - prev_siblings++; - } - } - prev_sibling = prev_sibling.getPreviousSibling(); - } - elementName = elementName.concat("[" + prev_siblings + "]"); - } - Node parent = node.getParentNode(); - if (parent == null) { - return xpath; - } - return getXPath(parent, "/" + elementName + xpath); - } + public static String buildXpathFromNode(Node node) { + return getXPath(node, ""); + } + + private static String getXPath(Node node, String xpath) { + String elementName = ""; + if (node instanceof Element) { + elementName = node.getNodeName(); + int prev_siblings = 1; + Node prev_sibling = node.getPreviousSibling(); + while (null != prev_sibling) { + if (prev_sibling.getNodeType() == node.getNodeType()) { + if (prev_sibling.getNodeName().equalsIgnoreCase(node.getNodeName())) { + prev_siblings++; + } + } + prev_sibling = prev_sibling.getPreviousSibling(); + } + elementName = elementName.concat("[" + prev_siblings + "]"); + } + Node parent = node.getParentNode(); + if (parent == null) { + return xpath; + } + return getXPath(parent, "/" + elementName + xpath); + } + + public static String getVTDXPath(VTDNav vn) { + + // Save off current navigator state + vn.push(); + String xp = ""; + + try { + // Move to a parent that is a node + while (vn.getTokenType(vn.getCurrentIndex()) != 0 && vn.getTokenType(vn.getCurrentIndex()) != 13) { + vn.toElement(VTDNav.PARENT); + } + + while (vn.getTokenType(vn.getCurrentIndex()) != 13) { + + // Find depth + int d = 1; + boolean e = vn.toElement(VTDNav.PREV_SIBLING); + while (e) { + d++; + e = vn.toElement(VTDNav.PREV_SIBLING); + } + + xp = "/" + vn.toNormalizedString(vn.getCurrentIndex()) + "[" + d + "]" + xp; + + vn.toElement(VTDNav.PARENT); + } + } catch (Exception e) { + return "Can not determine xpath address. Error:" + e.getMessage(); + } + + // Restore the navigator + vn.pop(); + + return xp; + } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/NodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/NodeValidator.java index 2018381..872f390 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/NodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/NodeValidator.java @@ -1,12 +1,15 @@ package org.sitenv.vocabularies.validation.validators; +import java.util.List; + import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.validation.NodeValidation; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; -import java.util.List; +import com.ximpleware.VTDNav; /** * Created by Brian on 2/15/2016. @@ -17,9 +20,9 @@ protected VocabularyValidationResult valuesetNotLoadedResult(NodeValidationResul VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - vocabularyValidationResult.setMessage("Value set code validation attempt for value set(s) ('" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ") that do not exist in service for code system " + nodeValidationResult.getRequestedCodeSystemName() + " (" + nodeValidationResult.getRequestedCodeSystem() + ")"); + vocabularyValidationResult.setMessage("Value set code validation attempt for value set(s) ('" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + "') that do not exist in service for code system " + nodeValidationResult.getRequestedCodeSystemName() + " (" + nodeValidationResult.getRequestedCodeSystem() + ")"); return vocabularyValidationResult; } - protected abstract List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredValidationResultSeverityLevel); + protected abstract List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredValidationResultSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav); } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java index 80f948b..3af676f 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java @@ -1,63 +1,69 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; -import org.sitenv.vocabularies.validation.utils.XpathUtils; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "ClassCodeValidator") public class ClassCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ClassCodeValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; - @Autowired - public ClassCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } + @Autowired + ValueSetDAO vsacValuesSetRepository; +// @Autowired +// public ClassCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } + @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String classCode; - try{ - XPathExpression exp = xpath.compile("@classCode"); - classCode = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + try { + classCode = vn.getAttrVal("classCode")!=-1 ? vn.toString(vn.getAttrVal("classCode")).toUpperCase() : ""; + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); } - List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); + List allowedConfiguredCodeSystemOids = new ArrayList<>( + Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); - nodeValidationResult.setRequestedClassCode(classCode); - nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); - if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ - nodeValidationResult.setNodeValuesetsFound(true); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); + nodeValidationResult.setRequestedClassCode(classCode); + nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); + + if (vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)) { + nodeValidationResult.setNodeValuesetsFound(true); if (vsacValuesSetRepository.codeExistsInValueset(classCode, allowedConfiguredCodeSystemOids)) { - nodeValidationResult.setValid(true); + nodeValidationResult.setValid(true); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult,configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java index 4c1dda4..f066749 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java @@ -1,65 +1,68 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.CodeRepository; +//import org.sitenv.vocabularies.validation.repositories.CodeRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "CodeSystemCodeValidator") public class CodeSystemCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(CodeSystemCodeValidator.class); - private CodeRepository codeRepository; - +// private CodeRepository codeRepository; @Autowired - public CodeSystemCodeValidator(CodeRepository codeRepository) { - this.codeRepository = codeRepository; - } + CodeSystemCodeDAO codeRepository; + +// @Autowired +// public CodeSystemCodeValidator(CodeRepository codeRepository) { +// this.codeRepository = codeRepository; +// } @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String nodeCode; String nodeCodeSystem; String nodeCodeSystemName; String nodeDisplayName; - try{ - XPathExpression expCode = xpath.compile("@code"); - XPathExpression expCodeSystem = xpath.compile("@codeSystem"); - XPathExpression expCodeSystemName = xpath.compile("@codeSystemName"); - XPathExpression expDisplayName = xpath.compile("@displayName"); - nodeCode = ((String) expCode.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystem = ((String) expCodeSystem.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystemName = ((String) expCodeSystemName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeDisplayName = ((String) expDisplayName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { - throw new RuntimeException("ERROR getting node values " + e.getMessage()); + try { + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + nodeCodeSystem = vn.getAttrVal("codeSystem")!=-1 ? vn.toString(vn.getAttrVal("codeSystem")).toUpperCase() : ""; + nodeCodeSystemName = vn.getAttrVal("codeSystemName")!=-1 ? vn.toString(vn.getAttrVal("codeSystemName")).toUpperCase() : ""; + nodeDisplayName = vn.getAttrVal("displayName")!=-1 ? vn.toString(vn.getAttrVal("displayName")).toUpperCase() : ""; + } catch (Exception e) { + throw new RuntimeException("ERROR getting node values " + e.getMessage()); + } - + List allowedConfiguredCodeSystemNames = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedCodesystemNames().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setRequestedCodeSystemName(nodeCodeSystemName); nodeValidationResult.setRequestedCodeSystem(nodeCodeSystem); nodeValidationResult.setRequestedDisplayName(nodeDisplayName); nodeValidationResult.setConfiguredAllowableCodesystemNamesForNode(configuredValidator.getAllowedCodesystemNames()); + nodeValidationResult.setRuleID(configuredValidator.getId()); if(codeRepository.foundCodeAndDisplayNameAndCodeSystemOIDInCodesystem(nodeCode, nodeDisplayName, nodeCodeSystem, allowedConfiguredCodeSystemNames)){ nodeValidationResult.setValid(true); @@ -77,38 +80,35 @@ public List validateNode(ConfiguredValidator configu } } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel){ + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav){ List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isCodeSystemFound()) { - if (!nodeValidationResult.isNodeCodeFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); - String validationMessage = "Code: " + nodeValidationResult.getRequestedCode() + " , Code System: " + nodeValidationResult.getRequestedCodeSystem() + " are not found in the configured code system name(s) " + nodeValidationResult.getConfiguredAllowableCodesystemNamesForNode(); - vocabularyValidationResult.setMessage(validationMessage); - vocabularyValidationResults.add(vocabularyValidationResult); - } - if (!nodeValidationResult.isNodeDisplayNameFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - String validationMessage = "Display Name " + nodeValidationResult.getRequestedDisplayName() + " does not exist in the code system " + nodeValidationResult.getRequestedCodeSystem() + " in the configured code system name(s) " + nodeValidationResult.getConfiguredAllowableCodesystemNamesForNode(); - vocabularyValidationResult.setMessage(validationMessage); - vocabularyValidationResults.add(vocabularyValidationResult); - } - if (!nodeValidationResult.isNodeCodeSystemOIDFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - String validationMessage = "Code system OID " + nodeValidationResult.getRequestedCodeSystem() + " does not exist in the configured code system name(s) " + nodeValidationResult.getConfiguredAllowableCodesystemNamesForNode(); - vocabularyValidationResult.setMessage(validationMessage); - vocabularyValidationResults.add(vocabularyValidationResult); - } - }else{ + if (!nodeValidationResult.isNodeCodeFound()) { + VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); + vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); + vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); + String validationMessage = "Code " + nodeValidationResult.getRequestedCode() + " does not exist in the code system " + nodeValidationResult.getRequestedCodeSystem() + " in the configured code system name(s) " + nodeValidationResult.getConfiguredAllowableCodesystemNamesForNode(); + vocabularyValidationResult.setMessage(validationMessage); + vocabularyValidationResults.add(vocabularyValidationResult); + } + if (filter.isEnabled(VocabularyValidationResultLevel.MAY)) { + if (!nodeValidationResult.isNodeDisplayNameFound()) { + VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); + vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); + vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); + String validationMessage = "Display Name " + nodeValidationResult.getRequestedDisplayName() + " does not exist in the code system " + nodeValidationResult.getRequestedCodeSystem() + " in the configured code system name(s) " + nodeValidationResult.getConfiguredAllowableCodesystemNamesForNode(); + vocabularyValidationResult.setMessage(validationMessage); + vocabularyValidationResults.add(vocabularyValidationResult); + } + } + }else { + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java index 4e499ce..bdbac0f 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java @@ -1,53 +1,58 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "LanguageCodeNodeCountryCodeValuesetValidator") public class LanguageCodeNodeCountryCodeValuesetValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(LanguageCodeNodeCountryCodeValuesetValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; - @Autowired - public LanguageCodeNodeCountryCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } + @Autowired + ValueSetDAO vsacValuesSetRepository; + +// @Autowired +// public LanguageCodeNodeCountryCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { - String nodeCode; - try{ - XPathExpression exp = xpath.compile("@code"); - nodeCode = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { + String nodeCode; + try { + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); - } - + + } List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); if(needToValidateCountryCode(nodeCode)){ nodeCode = StringUtils.substringAfter(nodeCode, "-"); @@ -61,25 +66,27 @@ public List validateNode(ConfiguredValidator configu nodeValidationResult.setValid(true); } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); - } + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); + } private boolean needToValidateCountryCode(String nodeCode) { return StringUtils.contains(nodeCode, "-"); } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); - String validationMessage = "Code '" + nodeValidationResult.getRequestedCode() + " (from " + nodeValidationResult.getRequestedCode()+ ")' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; + String validationMessage = "Code '" + nodeValidationResult.getRequestedCode() + "' (from " + nodeValidationResult.getRequestedCode()+ ")' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; vocabularyValidationResult.setMessage(validationMessage); vocabularyValidationResults.add(vocabularyValidationResult); }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java index c444f2d..0191e7e 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java @@ -1,53 +1,59 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +//import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "LanguageCodeNodeLanguageCodeValuesetValidator") public class LanguageCodeNodeLanguageCodeValuesetValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(LanguageCodeNodeLanguageCodeValuesetValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; - @Autowired - public LanguageCodeNodeLanguageCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } + @Autowired + ValueSetDAO vsacValuesSetRepository; + +// @Autowired +// public LanguageCodeNodeLanguageCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { - String nodeCode; - try{ - XPathExpression exp = xpath.compile("@code"); - nodeCode = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { + String nodeCode; + try { + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); - } - + } + List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ nodeValidationResult.setNodeValuesetsFound(true); @@ -58,21 +64,23 @@ public List validateNode(ConfiguredValidator configu nodeValidationResult.setValid(true); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); - } + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); + } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); - String validationMessage = "Code '" + nodeValidationResult.getRequestedCode() + " (from " + nodeValidationResult.getRequestedCode()+ ")' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; + String validationMessage = "Code '" + nodeValidationResult.getRequestedCode() + "' (from " + nodeValidationResult.getRequestedCode()+ ")' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; vocabularyValidationResult.setMessage(validationMessage); vocabularyValidationResults.add(vocabularyValidationResult); }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java index d575b7d..23926d7 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java @@ -1,71 +1,74 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.NodeValidation; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.stereotype.Component; +import org.w3c.dom.Element; import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "NodeCodeSystemMatchesConfiguredCodeSystemValidator") public class NodeCodeSystemMatchesConfiguredCodeSystemValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(NodeCodeSystemMatchesConfiguredCodeSystemValidator.class); @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String nodeCode; String nodeCodeSystem; String nodeCodeSystemName; String nodeDisplayName; - try{ - XPathExpression expCode = xpath.compile("@code"); - XPathExpression expCodeSystem = xpath.compile("@codeSystem"); - XPathExpression expCodeSystemName = xpath.compile("@codeSystemName"); - XPathExpression expDisplayName = xpath.compile("@displayName"); - nodeCode = ((String) expCode.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystem = ((String) expCodeSystem.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystemName = ((String) expCodeSystemName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeDisplayName = ((String) expDisplayName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { - throw new RuntimeException("ERROR getting node values " + e.getMessage()); - } + try{ + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + nodeCodeSystem = vn.getAttrVal("codeSystem")!=-1 ? vn.toString(vn.getAttrVal("codeSystem")).toUpperCase() : ""; + nodeCodeSystemName = vn.getAttrVal("codeSystemName")!=-1 ? vn.toString(vn.getAttrVal("codeSystemName")).toUpperCase() : ""; + nodeDisplayName = vn.getAttrVal("displayName")!=-1 ? vn.toString(vn.getAttrVal("displayName")).toUpperCase() : ""; + } catch (Exception e) { + throw new RuntimeException("ERROR getting node values " + e.getMessage()); + } List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setRequestedCodeSystemName(nodeCodeSystemName); nodeValidationResult.setRequestedCodeSystem(nodeCodeSystem); nodeValidationResult.setRequestedDisplayName(nodeDisplayName); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); for(String allowedConfiguredCodeSystemOid : allowedConfiguredCodeSystemOids){ if (nodeCodeSystem.equalsIgnoreCase(allowedConfiguredCodeSystemOid)) { nodeValidationResult.setValid(true); - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.SHALL); String validationMessage = "Code system '" + nodeValidationResult.getRequestedCodeSystem()+ "' is not valid for the node found for (" + nodeValidationResult.getValidatedDocumentXpathExpression() + ")"; diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java index ac8fa5d..fbfce29 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java @@ -1,73 +1,81 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "TextNodeValidator") public class TextNodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(TextNodeValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; - @Autowired - public TextNodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } + @Autowired + ValueSetDAO vsacValuesSetRepository; + +// @Autowired +// public TextNodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { - String nodeText; - try{ - XPathExpression exp = xpath.compile("text()"); - nodeText = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { + String nodeText; + try { + nodeText = vn.toString(vn.getText()).toUpperCase(); + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); - } + } - List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); + List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedText(nodeText); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); + if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ nodeValidationResult.setNodeValuesetsFound(true); if (vsacValuesSetRepository.codeExistsInValueset(nodeText, allowedConfiguredCodeSystemOids)) { nodeValidationResult.setValid(true); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } - + @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); String validationMessage = nodeValidationResult.getRequestedText() + "' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; vocabularyValidationResult.setMessage(validationMessage); vocabularyValidationResults.add(vocabularyValidationResult); }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java index e762748..886bd61 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java @@ -1,67 +1,74 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "UnitValidator") public class UnitValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(UnitValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; - - @Autowired - public UnitValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } +// private VsacValuesSetRepository vsacValuesSetRepository; + @Autowired + ValueSetDAO vsacValuesSetRepository; + +// @Autowired +// public UnitValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } + @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String nodeUnit; try{ - XPathExpression exp = xpath.compile("@unit"); - nodeUnit = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + nodeUnit = vn.getAttrVal("unit")!=-1 ? vn.toString(vn.getAttrVal("unit")).toUpperCase() : ""; + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); } List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedUnit(nodeUnit); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); + if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ nodeValidationResult.setNodeValuesetsFound(true); if (vsacValuesSetRepository.codeExistsInValueset(nodeUnit, allowedConfiguredCodeSystemOids)) { nodeValidationResult.setValid(true); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); if(nodeValidationResult.getRequestedUnit().indexOf('{') > -1){ vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.SHOULD); @@ -72,6 +79,7 @@ protected List buildVocabularyValidationResults(Node vocabularyValidationResult.setMessage(validationMessage); vocabularyValidationResults.add(vocabularyValidationResult); }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java index 12122ca..8453d0f 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java @@ -1,65 +1,66 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "ValueSetCodeValidator") public class ValueSetCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ValueSetCodeValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; + @Autowired + ValueSetDAO vsacValuesSetRepository; - @Autowired - public ValueSetCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } +// @Autowired +// public ValueSetCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String nodeCode; String nodeCodeSystem; String nodeCodeSystemName; String nodeDisplayName; - try{ - XPathExpression expCode = xpath.compile("@code"); - XPathExpression expCodeSystem = xpath.compile("@codeSystem"); - XPathExpression expCodeSystemName = xpath.compile("@codeSystemName"); - XPathExpression expDisplayName = xpath.compile("@displayName"); - nodeCode = ((String) expCode.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystem = ((String) expCodeSystem.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeCodeSystemName = ((String) expCodeSystemName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - nodeDisplayName = ((String) expDisplayName.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { - throw new RuntimeException("ERROR getting node values " + e.getMessage()); - } + try{ + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + nodeCodeSystem = vn.getAttrVal("codeSystem")!=-1 ? vn.toString(vn.getAttrVal("codeSystem")).toUpperCase() : ""; + nodeCodeSystemName = vn.getAttrVal("codeSystemName")!=-1 ? vn.toString(vn.getAttrVal("codeSystemName")).toUpperCase() : ""; + nodeDisplayName = vn.getAttrVal("displayName")!=-1 ? vn.toString(vn.getAttrVal("displayName")).toUpperCase() : ""; + } catch (Exception e) { + throw new RuntimeException("ERROR getting node values " + e.getMessage()); + } List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setRequestedCodeSystemName(nodeCodeSystemName); nodeValidationResult.setRequestedCodeSystem(nodeCodeSystem); nodeValidationResult.setRequestedDisplayName(nodeDisplayName); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ nodeValidationResult.setNodeValuesetsFound(true); @@ -80,42 +81,51 @@ public List validateNode(ConfiguredValidator configu } } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); } - - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel){ + + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav){ List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { if (!nodeValidationResult.isNodeCodeFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); - vocabularyValidationResult.setMessage("Code " + nodeValidationResult.getRequestedCode() + " does not exist in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); - vocabularyValidationResults.add(vocabularyValidationResult); - } - if (!nodeValidationResult.isNodeCodeSystemOIDFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - vocabularyValidationResult.setMessage("Code System " + nodeValidationResult.getRequestedCodeSystem() + " does not exist in value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); - vocabularyValidationResults.add(vocabularyValidationResult); - } - if (!nodeValidationResult.isNodeCodeSystemNameFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - vocabularyValidationResult.setMessage("Code System Name " + nodeValidationResult.getRequestedCodeSystemName() + " does not exist in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); + vocabularyValidationResult.setMessage("Code " + nodeValidationResult.getRequestedCode() + " does not exist in the code system " + nodeValidationResult.getRequestedCodeSystemName() + " (" + nodeValidationResult.getRequestedCodeSystem() + ") in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); vocabularyValidationResults.add(vocabularyValidationResult); } - if (!nodeValidationResult.isNodeDisplayNameFound()) { - VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); - vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); - vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); - vocabularyValidationResult.setMessage("Display Name " + nodeValidationResult.getRequestedDisplayName() + " does not exist in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); - vocabularyValidationResults.add(vocabularyValidationResult); + if (filter==null || filter.isEnabled(VocabularyValidationResultLevel.MAY)) { + if (!nodeValidationResult.isNodeCodeSystemOIDFound()) { + VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); + vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); + vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); + vocabularyValidationResult.setMessage("Code System " + nodeValidationResult.getRequestedCodeSystem() + " does not exist in value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); + vocabularyValidationResults.add(vocabularyValidationResult); + } + + if (!nodeValidationResult.isNodeCodeSystemNameFound()) { + VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); + vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); + vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); + vocabularyValidationResult.setMessage("Code System Name " + nodeValidationResult.getRequestedCodeSystemName() + " does not match expected name for the code system oid " + nodeValidationResult.getRequestedCodeSystem() + " in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); + vocabularyValidationResults.add(vocabularyValidationResult); + } + + if (!nodeValidationResult.isNodeDisplayNameFound()) { + VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); + vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); + vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.MAY); + vocabularyValidationResult.setMessage("Display Name " + nodeValidationResult.getRequestedDisplayName() + " does not exist in the code system " + nodeValidationResult.getRequestedCodeSystemName() + " (" + nodeValidationResult.getRequestedCodeSystem() + ") in the value set(s) " + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode()); + vocabularyValidationResults.add(vocabularyValidationResult); + } } }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java index 41cf185..02c0771 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java @@ -1,73 +1,81 @@ package org.sitenv.vocabularies.validation.validators.nodetypes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dao.ValueSetDAO; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; -import org.sitenv.vocabularies.validation.repositories.VsacValuesSetRepository; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.w3c.dom.Node; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.ximpleware.VTDNav; @Component(value = "ValueSetNodeWithOnlyCodeValidator") public class ValueSetNodeWithOnlyCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ValueSetNodeWithOnlyCodeValidator.class); - private VsacValuesSetRepository vsacValuesSetRepository; +// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired - public ValueSetNodeWithOnlyCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { - this.vsacValuesSetRepository = vsacValuesSetRepository; - } + ValueSetDAO vsacValuesSetRepository; + +// @Autowired +// public ValueSetNodeWithOnlyCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { +// this.vsacValuesSetRepository = vsacValuesSetRepository; +// } - @Override - public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, Node node, int nodeIndex) { + @Override + public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, + VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { String nodeCode; try{ - XPathExpression exp = xpath.compile("@code"); - nodeCode = ((String) exp.evaluate(node, XPathConstants.STRING)).toUpperCase(); - } catch (XPathExpressionException e) { + nodeCode = vn.getAttrVal("code")!=-1 ? vn.toString(vn.getAttrVal("code")).toUpperCase() : ""; + } catch (Exception e) { throw new RuntimeException("ERROR getting node values " + e.getMessage()); } List allowedConfiguredCodeSystemOids = new ArrayList<>(Arrays.asList(configuredValidator.getAllowedValuesetOids().split(","))); NodeValidationResult nodeValidationResult = new NodeValidationResult(); - nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.buildXpathFromNode(node)); + nodeValidationResult.setErrorOffset(vn.getTokenOffset(vn.getCurrentIndex())); + nodeValidationResult.setValidatedDocumentXpathExpression(xpathExpression); nodeValidationResult.setRequestedCode(nodeCode); nodeValidationResult.setConfiguredAllowableValuesetOidsForNode(configuredValidator.getAllowedValuesetOids()); + nodeValidationResult.setRuleID(configuredValidator.getId()); + if(vsacValuesSetRepository.valuesetOidsExists(allowedConfiguredCodeSystemOids)){ nodeValidationResult.setNodeValuesetsFound(true); if (vsacValuesSetRepository.codeExistsInValueset(nodeCode, allowedConfiguredCodeSystemOids)) { nodeValidationResult.setValid(true); } } - return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel()); - } + return buildVocabularyValidationResults(nodeValidationResult, configuredValidator.getConfiguredValidationResultSeverityLevel(), filter, vn); + } @Override - protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel) { + protected List buildVocabularyValidationResults(NodeValidationResult nodeValidationResult, ConfiguredValidationResultSeverityLevel configuredNodeAttributeSeverityLevel, ConfiguredExpressionFilter filter, VTDNav nav) { List vocabularyValidationResults = new ArrayList<>(); if(!nodeValidationResult.isValid()) { if (nodeValidationResult.isNodeValuesetsFound()) { VocabularyValidationResult vocabularyValidationResult = new VocabularyValidationResult(); + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResult.setNodeValidationResult(nodeValidationResult); vocabularyValidationResult.setVocabularyValidationResultLevel(VocabularyValidationResultLevel.valueOf(configuredNodeAttributeSeverityLevel.getCodeSeverityLevel())); String validationMessage = "Code '" + nodeValidationResult.getRequestedCode()+ "' does not exist in the value set (" + nodeValidationResult.getConfiguredAllowableValuesetOidsForNode() + ")"; vocabularyValidationResult.setMessage(validationMessage); vocabularyValidationResults.add(vocabularyValidationResult); }else{ + nodeValidationResult.setValidatedDocumentXpathExpression(XpathUtils.getVTDXPath(nav)); vocabularyValidationResults.add(valuesetNotLoadedResult(nodeValidationResult)); } } diff --git a/src/main/resources/CodeValidator.properties b/src/main/resources/CodeValidator.properties new file mode 100644 index 0000000..9ecc750 --- /dev/null +++ b/src/main/resources/CodeValidator.properties @@ -0,0 +1,8 @@ +executor.maxPoolSize=400 +executor.corePoolSize=200 +executor.queueCapacity=20000 + +dataSource.initialSize=10 +dataSource.minIdle=10 +dataSource.maxActive=20 + diff --git a/src/main/resources/ais.properties b/src/main/resources/ais.properties new file mode 100644 index 0000000..9ecc750 --- /dev/null +++ b/src/main/resources/ais.properties @@ -0,0 +1,8 @@ +executor.maxPoolSize=400 +executor.corePoolSize=200 +executor.queueCapacity=20000 + +dataSource.initialSize=10 +dataSource.minIdle=10 +dataSource.maxActive=20 + diff --git a/src/main/resources/schema.sql b/src/main/resources/schema.sql index 33fe126..316bb7f 100644 --- a/src/main/resources/schema.sql +++ b/src/main/resources/schema.sql @@ -1,5 +1,3 @@ -DROP SCHEMA PUBLIC CASCADE; -COMMIT; CREATE TABLE VALUESETS ( ID INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, diff --git a/src/test/java/com/cerner/hs/ccda/validator/mu2ccdaDocumentSample.xml b/src/test/java/com/cerner/hs/ccda/validator/mu2ccdaDocumentSample.xml new file mode 100644 index 0000000..9e4e2fa --- /dev/null +++ b/src/test/java/com/cerner/hs/ccda/validator/mu2ccdaDocumentSample.xml @@ -0,0 +1,2741 @@ + + + + + + + + + Transition of Care474 + + + + + + + + + + + + 428 Apple Blvd + + Malvern + PA + 124 + United States of America + + + + + + + DevTest + + VK + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CentraState Med Ctr - DRD + + + + 51 Valley Stream Parkway + + Malvern + PA + 19355 + 0108081206Country + + + Secondary Address + + Malvern + PA + 123456 + United States of America + + + + + + + + + + + CentraState Healthcare System + + + 51 Valley Stream Parkway + + Malvern + PA + 19355 + 0108081206Country + + + + + + + + + + + + + + + Primary Care Provider + + + + + 65 Valley Stream Parkway Malvern PA + + + + + + + + + + + PrimaryFirst + PrimaryLast + + + + + + + + + + + + + + + + + + 51 Valley Stream Parkway Malvern PA + + + + + + + + + + + + RefferingFirst + RefferingLast + + + + + + + + + 51 Valley Stream Parkway + + Malvern + PA + 19355 + 0108081206Country + + + Secondary Address + + Malvern + PA + 123456 + United States of America + + + + + + MD + Beverly + Coats + + + + + + + + + 420 Coleman Street + + AllenTown + PA + 23332 + United States of America + + + + + + MD + Dori + Coleman + + + + + + + + + 51 Valley Stream Parkway + + Malvern + PA + 19355 + 0108081206Country + + + Secondary Address + + Malvern + PA + 123456 + United States of America + + + + + + + Danny + Edison + + + + + + + + + 418 Malberry street + + Bryn Mawr + PA + 12345 + United States of America + + + Secondary Office Street + + Exton + PA + 34344 + United States of America + + + + + + + DO + Rachel + Allred + + + + + + + + + 412 MacAurthur Rd + + DowningTown + PA + 23455 + United States of America + + + Secondary Street Address + + Exton + PA + 23222 + United States of America + + + + + + + + S + Brown + + + + + + + + + 2CCU DRD + + + + + + + + + + + +
+ + + + Problems + + + Encounter Diagnosis +
+ + + Acute Pain Acute/Chronic:Acute, Onset:Today, Rank:2 - Medium, Severity:Mild, Status:Active. + + + Bradycardia Rank:1 - High, Rank:1, Status:Inactive. + + + Chronic Pain Status:Active. + + + Hyperglycemia Status:Active. + + + Pain Onset Date:2012-03-06 00:00:00, Status:Active. + + + Additional Problems +
+ + + Acute Confusion + + Status:Active. + + + + Chronic Sadness Onset:Childbirth, Onset Date:2012-11-06 00:00:00, Rank:2 - Medium, Status:Active. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + Encounters + + + Encounter Diagnosis +
+ + + Acute Pain Acute/Chronic:Acute, Onset:Today, Rank:2 - Medium, Severity:Mild, Status:Active. + + + Bradycardia Rank:1 - High, Rank:1, Status:Inactive. + + + Chronic Pain Status:Active. + + + Hyperglycemia Status:Active. + + + Pain Onset Date:2012-03-06 00:00:00, Status:Active. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + Allergies, Adverse Reactions, Alerts + + + + + Almond (as Food allergen) causes Moderate MikeAlgReact and cold sweats. Onset childhood. + + + Milk causes childhood. + + + Milk (as Food allergen) causes childhood. + + + Latex causes Mild Dizziness and Mild Fever. + + + Bactrim causes Mild Diarrhea and Moderate Rash. + + + Percocet causes Mild Dizziness. + + + Ampicillin causes Moderate TEST REACTION FROM VK. Onset childhood. + + + No IV Contrast Allergy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + Medication + + + It is the responsibility of the patient or patient representative to confirm the list of medications with either the patient's personal care provider or the patient's follow-up care provider to ensure the patient has an appropriate list of medications to take at home. +
+ Preliminary list - medication reconciliation not completed. + + Discharge medications + + New medications + + + ibuprofen 200 mg Capsule, Ordered By: Thomas Anderson
Directions: 2 capsule oral twice a day after meals + + + + + +
+
+ + + aliskiren-amlodipine (Tekamlo) 150 mg-10 mg Tablet, Ordered By: Thomas Anderson
Directions: 1 tablet jejunostomy three times a day + +
Additional Instructions: Patient is higher fewer + + + + + +
+
+ + + + Stopped medications + + + + None + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tekamlo + + + + + + + + + + + + + + + + + +
+ + +
+ + + + Immunizations + + + + No immunizations administered or ordered. + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + Social History + + + Social History from 8/29/2013 1:50 PM: + + Personal Habits +
+ + SNOMED Smoking Status : Current Some Day Smoker + + + Social History from 4/3/2013 7:30 AM: + + Personal Habits +
+ + Tobacco Use? : Yes + +
+ + + + + + + + + + + + + + + +
+ + +
+ + + Hospital Discharge Instructions + + + Discharge Instructions - All from 10/15/2012 12:06 PM: + + Notify Physician If: +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions - All from 8/24/2012 4:29 PM: + + Activity and Other Restrictions +
+ + Activity : Bed Rest,OOB to Chair,Up ad lib + + + Stairs : As tolerated,1 Flight + + + Bathing : Keep Sutures Dry + + Equiipment +
+ + Cane : Yes + + + Cane Provider Phone # : test + + Notify Physician If: +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions - All from 7/31/2012 5:03 PM: + + Activity and Other Restrictions +
+ + Activity : Bed Rest,OOB to Chair,OOB with Assist + + + Stairs : As tolerated,2 Flights + + + Driving : Day Only + + Notify Physician If: +
+ + Notify Physician For : Fever or chills Nausea and /or vomiting greater than 3 times or for longer than 24 hours, not relieved with anti-nausea medication Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours Constipation, no bowel movement in 3 days or longer Shortness of breath Chest pain Redness, swelling, warmth, pain at IV access site Redness, swelling, warmth around incision Bleeding or drainage from incision Severe weakness or tiredness Severe pain that pain medication does not control + +
+ +
+ + +
+ + + Plan of Care + + + Discharge Instructions from 10/15/2012 12:06 PM: + + Notify Physician For +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions from 8/24/2012 4:29 PM: + + Notify Physician For +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions from 7/31/2012 5:03 PM: + + Notify Physician For +
+ + Notify Physician For : Fever or chills Nausea and /or vomiting greater than 3 times or for longer than 24 hours, not relieved with anti-nausea medication Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours Constipation, no bowel movement in 3 days or longer Shortness of breath Chest pain Redness, swelling, warmth, pain at IV access site Redness, swelling, warmth around incision Bleeding or drainage from incision Severe weakness or tiredness Severe pain that pain medication does not control + + + Discharge Instructions - All from 10/15/2012 12:06 PM: + + Notify Physician If: +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions - All from 8/24/2012 4:29 PM: + + Activity and Other Restrictions +
+ + Activity : Bed Rest,OOB to Chair,Up ad lib + + + Stairs : As tolerated,1 Flight + + + Bathing : Keep Sutures Dry + + Equiipment +
+ + Cane : Yes + + + Cane Provider Phone # : test + + Notify Physician If: +
+ + Notify Physician For : Fever or chills; Nausea and/or vomiting greater than 3 time or for longer than 24 hours, not relieved with anti-nausea medication; Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours; Constipation, no bowel movement in 3 days or longer; Shortness of breath; Chest pain; Redness, swelling, warmth, pain at IV access site; Redness, swelling, warmth around incision, bleeding or drainage from incision; Severe weakness or tiredness; Severe pain that pain medication does not control + + + Discharge Instructions - All from 7/31/2012 5:03 PM: + + Activity and Other Restrictions +
+ + Activity : Bed Rest,OOB to Chair,OOB with Assist + + + Stairs : As tolerated,2 Flights + + + Driving : Day Only + + Notify Physician If: +
+ + Notify Physician For : Fever or chills Nausea and /or vomiting greater than 3 times or for longer than 24 hours, not relieved with anti-nausea medication Diarrhea in large quantity or more than 3-4 bowel movements in 24 hours Constipation, no bowel movement in 3 days or longer Shortness of breath Chest pain Redness, swelling, warmth, pain at IV access site Redness, swelling, warmth around incision Bleeding or drainage from incision Severe weakness or tiredness Severe pain that pain medication does not control + +
+ +
+ + +
+ + + Functional Status + + + CLX Admission 3.4 Flowsheet from 8/29/2013 1:50 PM: + + CLX 3.4 Admission +
+ + Clothing : Yes + + + CLX Admission 3.4 Flowsheet from 11/27/2012 11:59 AM: + + CLX 3.4 Admission +
+ + Source of Info : Self + + + Accompanied By : Spouse + + + Mode/Arrival : Ambulatory/Carry + + + Source Reliable? : Yes + + + CLX Admission 3.4 Flowsheet from 11/19/2012 8:17 AM: + + CLX 3.4 Admission +
+ + Legal Guardian : VK + + + Source of Info : Spouse + + + Accompanied By : Spouse,Significant Other + + + Mode/Arrival : Cart/Crib + + + CLX Admission 3.4 Flowsheet from 8/5/2012 4:45 PM: + + CLX 3.4 Admission +
+ + Accompanied By : Spouse,Parent + +
+ +
+ + +
+ + + Reason For Visit + + + Reason for Visit from 11/27/2012 1:35 PM: + + Reason for Visit +
+ + Pt Stated Reason for Adm : because I am sick + + + Reason for Visit from 11/27/2012 11:59 AM: + + Reason for Visit +
+ + Pt Stated Reason for Adm : Rockin' pneumonia + + +
+ + +
+ + + Chief Complaint + Line 1
+
Line 2
+
Line 3
Line 4
+
Line 5
+
Line 6 +
+ + +
+ + + Hospital Course + +
+ + +
+ + + + Procedures + + + + No relevant procedures performed. + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + Results + + + HEMATOLOGY from 6/12/2013 1:58 PM + + + WHITE BLOOD COUNT 5 % + + HEMATOLOGY from 12/10/2012 10:35 AM + + + WHITE BLOOD COUNT 6009 (Preliminary Result) + + HEMATOLOGY from 10/26/2012 3:53 PM + + + HEMATOCRIT 55 % + + HEMATOLOGY from 10/8/2012 3:01 PM + + + HEMATOCRIT 55 % + + CHEMISTRY from 12/10/2012 10:36 AM + + + GLUCOSE Noraml (Preliminary Result) + + CHEMISTRY from 12/5/2012 2:44 PM + + + GLUCOSE 50 + + CHEMISTRY from 9/28/2012 10:30 AM + + + SODIUM LEVEL 10 % A +
+ + GLUCOSE 5 % + + CHEMISTRY from 9/20/2012 10:00 AM + + + SODIUM LEVEL Normal : 120 +
+ + POTASSIUM LEVEL 22 +
+ + GLUCOSE 96 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + VITAL SIGNS + + + + + + + + + + + + + + + + + + + + + + + + + + +
Date / Time: Sept 10, 2012Sept 1, 2011
Height177 cm177 cm
Weight86 kg88 kg
Blood Pressure132/88128/80
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + diff --git a/src/test/java/com/cerner/hs/ccda/validator/mu3ccdaDocumentSample.xml b/src/test/java/com/cerner/hs/ccda/validator/mu3ccdaDocumentSample.xml new file mode 100644 index 0000000..994ce91 --- /dev/null +++ b/src/test/java/com/cerner/hs/ccda/validator/mu3ccdaDocumentSample.xml @@ -0,0 +1,882 @@ + + + + + + + + + Discharge Summary3 + + + + + + + + 2020 Anderson Ct + + Malvern + PA + 17105 + United States + + + + + + Care Providers + + Lysko + + + + + + + + + + + + + + + + + + + + + + + + City General + + + 105 N 9th St + + Philadelphia + MN + 19100 + United States + + + + + + + + + + + Health Enterprise + + + 53 Valley Stream Parkway + + Malvern + PA + 19355 + United States + + + + + + + + + + + + + + + Primary Care Provider-() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1174 Schuylkill Hwy + + Phoenixville + + 19033 + + + + + + + Rachel + Allred + + + + + + + + + + 105 N 9th St + + Philadelphia + MN + 19100 + United States + + + + + + Joseph + Barclay + + + + + + + + + + + + + + Sobrina-INT + Butler + + + + + + + + + + 459 + + Default Address + + + + + + + + + + + + + + + + + +
+ + + + Hospital Course + +
+ + +
+ + + + + Hospital Discharge Diagnosis + + + + + + + + Phantom limb syndrome without pain  Code: G54.7  Status: ACTIVE  Priority: Primary  Start Date: 4/5/2016 11:54 AM   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + Problems + + + Encounter Diagnosis +
+ + + Abdominal Wall Pain Status:Active. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + Allergies and Intolerances + + This section is representative of the current allergy information, at the time of the CCD generation. In the case of regeneration of the CCD, the allergy information may not reflect the state of known allergies at the time of the CCD's subject visit.
+
+ + + No Latex Allergy. + + + No IV Contrast Allergy. + + + No Known Drug Allergies. + + + No Known Food Allergies. + + + No Known Allergies. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + Immunizations + + + + No immunizations administered or ordered. + + + +
+ + +
+ + + + + + Medication + + + + Medication reconciliation has not been performed. + + + +
+ + +
+ + + + + + Procedures + + + + No relevant procedures performed. + + + +
+ + +
+ + + Medical Equipment Section + + + + + + No implantable medical devices are associated with this visit. + + +
+ + +
+ + + + Social History + +
+ + + + + + + + + + + + + + +
+ + +
+ + + + Mental Status + +
+ +
+ + +
+ + + + Functional Status + +
+ +
+ + +
+ + + + + + Vital Signs + + + CLX_Vtitals from 4/5/2016 11:51 AM: + + + A_BP : 100/60 + + + A_Pulse : 19 + + + Respirations : 61 + + + A_Height : 2/ ft,in + + + A_Weight : 32/ lbs,oz + + + CLX_Vtitals from 4/5/2016 2:34 AM: + + + A_BP : 123/78 + + + A_Height : /97 ft,in + + + A_Weight : 78000/ g + + + CLX_Vtitals from 4/4/2016 5:30 AM: + + + A_BP : 123/83 + + + A_Pulse : 123 + + + A_Height : 3215/2.68 ft,in + + + A_Weight : 99000/ g + +
+ +
+ + +
+ + + Assessments + +
+ +
+ + +
+ + + + Treatment Plan + +
+ +
+ + +
+ + + Health Concerns + +
+ + + + + + + + + +
+ + +
+ + + Goals + +
+ +
+ + +
+ + + Hospital Discharge Instructions + +
+ +
+ + +
+ + + + + + Results + +
Lab results are displayed in the following format if available: Test Name, Test Value, Value Unit, Reference Range - (exception is microbiology results)
Example 1: Sodium 145 mEq/L (136-145 mEq/L) Example 2: Potassium: 3.4 mEq/L (Reference Range: not available)
+
+ +
+ + + + diff --git a/src/test/java/com/cerner/hs/ccda/validator/utils/ConfiguredExpressionLevelFilterTest.java b/src/test/java/com/cerner/hs/ccda/validator/utils/ConfiguredExpressionLevelFilterTest.java new file mode 100644 index 0000000..d290eb3 --- /dev/null +++ b/src/test/java/com/cerner/hs/ccda/validator/utils/ConfiguredExpressionLevelFilterTest.java @@ -0,0 +1,57 @@ +package com.cerner.hs.ccda.validator.utils; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; +import org.sitenv.vocabularies.configuration.ConfiguredExpression; +import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; +import org.sitenv.vocabularies.configuration.ConfiguredValidator; +import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; +import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionLevelFilter; + +import static org.junit.Assert.assertTrue; + +public class ConfiguredExpressionLevelFilterTest { + + @Test + public void testInfoLevel() { + ConfiguredExpressionLevelFilter f = new ConfiguredExpressionLevelFilter("info"); + ConfiguredExpression e = new ConfiguredExpression(); + e.setConfiguredXpathExpression("new xpath expression"); + List vs = new ArrayList(); + e.setConfiguredValidators(vs); + ConfiguredValidator v = new ConfiguredValidator(); + vs.add(v); + ConfiguredValidationResultSeverityLevel lvl = new ConfiguredValidationResultSeverityLevel(); + lvl.setCodeSeverityLevel(VocabularyValidationResultLevel.SHOULD.toString()); + v.setConfiguredValidationResultSeverityLevel(lvl); + + ConfiguredExpression r = f.accept(e); + assertTrue("Since level is info, return same object", r == e); + } + + @Test + public void testWarningLevel() { + ConfiguredExpressionLevelFilter f = new ConfiguredExpressionLevelFilter("error"); + ConfiguredExpression e = new ConfiguredExpression(); + e.setConfiguredXpathExpression("new xpath expression"); + List vs = new ArrayList(); + e.setConfiguredValidators(vs); + + ConfiguredValidator v1 = new ConfiguredValidator(); + vs.add(v1); + ConfiguredValidationResultSeverityLevel lvl1 = new ConfiguredValidationResultSeverityLevel(); + lvl1.setCodeSeverityLevel(VocabularyValidationResultLevel.MAY.toString()); + v1.setConfiguredValidationResultSeverityLevel(lvl1); + + ConfiguredValidator v2 = new ConfiguredValidator(); + vs.add(v2); + ConfiguredValidationResultSeverityLevel lvl2 = new ConfiguredValidationResultSeverityLevel(); + lvl2.setCodeSeverityLevel(VocabularyValidationResultLevel.SHALL.toString()); + v2.setConfiguredValidationResultSeverityLevel(lvl2); + + ConfiguredExpression r = f.accept(e); + assertTrue("No validators qualify", r.getConfiguredValidators().size() == 1); + } +} From 5cf06cf9af1bff812f528dc1271efd1b73b96363 Mon Sep 17 00:00:00 2001 From: Lakshmi Dintakurty Date: Tue, 25 Jul 2017 11:21:56 -0400 Subject: [PATCH 2/3] Cleanup DB after loading codes and valusets to HashSets. --- .../loader/VocabularyLoadRunner.java | 42 +++++++++++++++++++ .../validation/dao/CodeSystemCodeDAO.java | 15 ++++++- .../validation/dao/ValueSetDAO.java | 16 +++++++ src/main/resources/CodeValidator.properties | 1 + 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java index 2461157..8c8f495 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java +++ b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java @@ -17,10 +17,13 @@ import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; /** * Created by Brian on 2/6/2016. */ +@PropertySource("classpath:CodeValidator.properties") public class VocabularyLoadRunner implements InitializingBean, DisposableBean { private VocabularyLoaderFactory vocabularyLoaderFactory; private static Logger logger = Logger.getLogger(VocabularyLoadRunner.class); @@ -29,6 +32,9 @@ public class VocabularyLoadRunner implements InitializingBean, DisposableBean { private boolean recursive = true; private DataSource dataSource; + @Value("${cleanUpDatabaseAfterLoadingHashSets}") + private boolean cleanUpDatabaseAfterLoadingHashSets; + @Autowired ValueSetDAO vsdao; @@ -171,6 +177,34 @@ public void afterPropertiesSet() throws Exception { logger.error("Failed to load configured vocabulary directory.", e); }finally { try { + logger.info("!!!!*********** cleanUpDatabaseAfterLoadingHashSets falg set as : " + + cleanUpDatabaseAfterLoadingHashSets); + /* + * If the 'cleanUpDatabaseAfterLoadingHashSets' defined in + * CodeValidator.properties is set to true, then all the records + * from (Codes and ValueSets tables) the DB are deleted. + * + * Note: If there are any methods that are defined in the + * Repository classes (CodeRepository, VsacValuesSetRepository + * etc.,) are not implemented in the DAO classes (ValueSetDAO, + * CodeSystemDAO) and the same methods are being used by the + * consumer/caller then the flag + * cleanUpDatabaseAfterLoadingHashSets should be set to false. + * Otherwise if this flag was set to true and when these methods + * are invoked, no records will be returned. Ex., In the current + * versionm, these three methods findByCodeAndCodeSystemIn; + * findByValuesetOidIn, findByCodeAndValuesetOidIn defined in + * CodeRepository and ValueSetRepository are not implemented in + * ValueSetDAO & CodeSystemDAO classes. Hence, if any + * client/consumer wants to leverage these helper methods, + * should set the flag to false. + * + */ + + if(cleanUpDatabaseAfterLoadingHashSets) { + perfromDBCleanup(connection); + logger.info("!!!!*********** DB Cleanup completed **************!!!!"); + } if(connection != null && !(connection.isClosed())) { connection.close(); } @@ -180,6 +214,14 @@ public void afterPropertiesSet() throws Exception { } } + private void perfromDBCleanup(Connection connection) throws SQLException { + csdao.cleanupDBAfterLoadingToHashSets(dataSource); + connection.commit(); + + vsdao.cleanupDBAfterLoadingToHashSets(dataSource); + connection.commit(); + } + @Override public void destroy() throws Exception { logger.info("Destroying Loader Bean. Loading is done."); diff --git a/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java index 5472953..29b8b2d 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java +++ b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java @@ -13,7 +13,6 @@ import org.apache.log4j.Logger; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowCallbackHandler; -import org.springframework.jdbc.core.RowCountCallbackHandler; import org.springframework.stereotype.Component; @Component @@ -28,6 +27,7 @@ public class CodeSystemCodeDAO { private static Map> CodeSystemOIDToCodeSystems = new HashMap>(); private static boolean done = false; + private static boolean hasDBCleanupDone = false; private void addRow(String code, String codeSystem, String displayName, String codeSystemOID) { CodeSystems.add(codeSystem); @@ -35,6 +35,7 @@ private void addRow(String code, String codeSystem, String displayName, String c if (CodeToCodeSystems.get(code) == null) { CodeToCodeSystems.put(code, new HashSet()); } + CodeToCodeSystems.get(code).add(codeSystem); if (DisplayNameToCodeSystems.get(displayName) == null) { @@ -78,6 +79,17 @@ public synchronized void loadCodes(DataSource ds) { done = true; logger.info("Code Maps initialized. Total rows:" + handler.n); } + + public synchronized void cleanupDBAfterLoadingToHashSets(DataSource ds) { + if (hasDBCleanupDone) { + return; + } + String deleteCodesQry = "Delete FROM Codes"; + JdbcTemplate tmpl = new JdbcTemplate(ds); + tmpl.update(deleteCodesQry); + hasDBCleanupDone = true; + logger.info("*********** All Codes deleted from DB *************"); + } public class CodeSetRowHandler implements RowCallbackHandler { public long n = 0; @@ -199,4 +211,5 @@ public boolean foundCodeSystemOIDInCodesystems(String codeSystemOID, List> DisplayNameToValueSetOID = new HashMap>(); private static boolean done = false; + private static boolean hasDBCleanupDone = false; private void addRow(String code, String codeSystem, String displayName, String codeSystemName, String valueSetOID) { @@ -100,6 +104,17 @@ public synchronized void loadValueSets(DataSource ds) { logger.info("ValueSetOIDs:" + ValueSetOIDS.size()); } + public synchronized void cleanupDBAfterLoadingToHashSets(DataSource ds) { + if (hasDBCleanupDone) { + return; + } + String deleteValueSetsQry = "Delete FROM ValueSets"; + JdbcTemplate tmpl = new JdbcTemplate(ds); + tmpl.update(deleteValueSetsQry); + hasDBCleanupDone = true; + logger.info("*********** All ValueSets deleted from DB *************"); + } + public class ValueSetRowHandler implements RowCallbackHandler { public long n = 0; @Override @@ -268,4 +283,5 @@ public boolean codeExistsInValueset(String code, List valuesetOids) { } return false; } + } diff --git a/src/main/resources/CodeValidator.properties b/src/main/resources/CodeValidator.properties index 9ecc750..4002e6e 100644 --- a/src/main/resources/CodeValidator.properties +++ b/src/main/resources/CodeValidator.properties @@ -6,3 +6,4 @@ dataSource.initialSize=10 dataSource.minIdle=10 dataSource.maxActive=20 +cleanUpDatabaseAfterLoadingHashSets=TRUE From 7ad03e4ce5186de2faeff120e98471fdf46f021d Mon Sep 17 00:00:00 2001 From: Lakshmi Dintakurty Date: Fri, 28 Jul 2017 11:20:03 -0400 Subject: [PATCH 3/3] Code Cleanup - remove all commented out code --- .../CodeValidatorApiConfiguration.java | 37 +----- .../loader/VocabularyLoadRunner.java | 40 ------ .../vocabularies/loader/VocabularyLoader.java | 1 - .../vocabularies/loader/code/CdtLoader.java | 92 ------------- .../vocabularies/loader/code/CptLoader.java | 26 +--- .../loader/code/Icd10BaseLoader.java | 34 ----- .../loader/code/Icd9BaseLoader.java | 27 +--- .../vocabularies/loader/code/IcdLoader.java | 2 - .../vocabularies/loader/code/LoincLoader.java | 122 ++---------------- .../loader/code/RxNormLoader.java | 28 ---- .../loader/code/SnomedLoader.java | 33 ----- .../loader/valueset/VsacLoader.java | 32 ----- .../validation/NodeValidation.java | 8 +- .../validation/dao/CodeSystemCodeDAO.java | 5 - .../validation/dao/ValueSetDAO.java | 3 - .../validation/services/ValidateRequest.java | 1 - .../validation/services/ValidateWorker.java | 4 +- .../services/VocabularyValidationService.java | 14 -- .../nodetypes/ClassCodeValidator.java | 6 - .../nodetypes/CodeSystemCodeValidator.java | 6 - ...eCodeNodeCountryCodeValuesetValidator.java | 6 - ...CodeNodeLanguageCodeValuesetValidator.java | 5 - ...mMatchesConfiguredCodeSystemValidator.java | 3 - .../nodetypes/TextNodeValidator.java | 5 - .../validators/nodetypes/UnitValidator.java | 5 - .../nodetypes/ValueSetCodeValidator.java | 5 - .../ValueSetNodeWithOnlyCodeValidator.java | 6 - 27 files changed, 29 insertions(+), 527 deletions(-) diff --git a/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java b/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java index 0a6a2c9..b5c5ca8 100644 --- a/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java +++ b/src/main/java/org/sitenv/vocabularies/configuration/CodeValidatorApiConfiguration.java @@ -113,10 +113,7 @@ public EntityManagerFactory entityManagerFactory() { factory.setPackagesToScan("org.sitenv.vocabularies.validation.entities"); Properties jpaProperties = new Properties(); jpaProperties.put("hibernate.hbm2ddl.auto", "none"); - // jpaProperties.put("hibernate.dialect", - // "org.hibernate.dialect.HSQLDialect"); jpaProperties.put("hibernate.dialect", "org.hibernate.dialect.H2Dialect"); -// jpaProperties.put("hibernate.dialect", "org.hibernate.dialect.SQLServer2008Dialect"); jpaProperties.put("hibernate.format_sql", "true"); jpaProperties.put("hibernate.show_sql", "false"); jpaProperties.put("hibernate.connection.pool_size", "80"); @@ -171,9 +168,7 @@ public DataSource dataSource() { ds.setInitialSize(initialSize); ds.setMinIdle(minIdle); ds.setMaxTotal(maxActive); // DBCP2 maxTotal: The maximum number of active connections that can be allocated from this pool at the same time, or negative for no limit. - // ds.setDriverClassName("org.hsqldb.jdbcDriver"); ds.setDriverClassName("org.h2.Driver"); - //ds.setDriverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); return ds; } @@ -196,20 +191,6 @@ public VocabularyLoaderFactory vocabularyLoaderFactory() { return (VocabularyLoaderFactory) vocabularyLoaderFactoryServiceLocatorFactoryBean().getObject(); } - // @Bean - // public ServiceLocatorFactoryBean - // vocabularyValidatorFactoryServiceLocatorFactoryBean() { - // ServiceLocatorFactoryBean bean = new ServiceLocatorFactoryBean(); - // bean.setServiceLocatorInterface(VocabularyValidatorFactory.class); - // return bean; - // } - - // @Bean - // public VocabularyValidatorFactory vocabularyValidatorFactory() { - // return (VocabularyValidatorFactory) - // vocabularyValidatorFactoryServiceLocatorFactoryBean().getObject(); - // } - @Autowired @Bean VocabularyLoadRunner vocabularyLoadRunner(final Environment environment, @@ -240,19 +221,16 @@ VocabularyLoadRunner vocabularyLoadRunner(final Environment environment, * MU2 document validation. */ - // ------------------------- INTERNAL CODE CHANGE START - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE START -------------------------- /* * @Bean public List * vocabularyValidationConfigurations(ValidationConfigurationLoader * configurationLoader){ return * configurationLoader.getConfigurations().getExpressions(); } */ - // ------------------------- INTERNAL CODE CHANGE END - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE END -------------------------- - // ------------------------- INTERNAL CODE CHANGE START - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE START -------------------------- // @Bean // public DocumentBuilder documentBuilder() throws // ParserConfigurationException { @@ -268,8 +246,7 @@ public DocumentBuilderFactory documentBuilderFactory() throws ParserConfiguratio return domFactory; } - // ------------------------- INTERNAL CODE CHANGE END - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE END ------------------ @Bean public XPathFactory xPathFactory() { @@ -300,8 +277,7 @@ public Jaxb2Marshaller castorMarshaller() { * Following set of code is added for enhancing open source code base to * support MU2 document validation. */ - // ------------------------- INTERNAL CODE CHANGE START - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE START -------------------------- @Bean public List vocabularyValidationConfigurations( @@ -341,6 +317,5 @@ public List vocabularyValidationConfigurationsForMu2( return mu2Expressions; } - // ------------------------- INTERNAL CODE CHANGE END - // -------------------------- + // ------------------------- INTERNAL CODE CHANGE END -------------------------- } diff --git a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java index 8c8f495..8db1f34 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java +++ b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoadRunner.java @@ -41,8 +41,6 @@ public class VocabularyLoadRunner implements InitializingBean, DisposableBean { @Autowired CodeSystemCodeDAO csdao; -// @Autowired -// EntityManagerFactory entityManagerFactory; public void loadDirectory(String directory, DataSource datasource) throws IOException { File dir = new File(directory); @@ -56,17 +54,6 @@ public void loadDirectory(String directory, DataSource datasource) throws IOExce } } -// public void loadDirectory(String directory, Connection connection) throws IOException { -// File dir = new File(directory); -// if (dir.isFile()) { -// throw new IOException("Directory to Load is a file and not a directory"); -// } else { -// File[] list = dir.listFiles(); -// for (File file : list) { -// load(file, connection); -// } -// } -// } private void load(File directory, DataSource datasource) throws IOException { if (directory.isDirectory() && !directory.isHidden()) { @@ -84,21 +71,6 @@ private void load(File directory, DataSource datasource) throws IOException { } } -// private void load(File directory, Connection connection) throws IOException { -// if (directory.isDirectory() && !directory.isHidden()) { -// File[] filesToLoad = directory.listFiles(); -// logger.debug("Building Loader for directory: " + directory.getName() + "..."); -// VocabularyLoader loader = vocabularyLoaderFactory.getVocabularyLoader(directory.getName()); -// if (loader != null && filesToLoad != null) { -// logger.debug("Loader built..."); -// logger.info("Loading files in : " + directory.getName() + "..."); -// loader.load(Arrays.asList(filesToLoad), connection); -// logger.debug("File loaded..."); -// } else { -// logger.debug("Building of Loader Failed."); -// } -// } -// } public String getCodeDirectory() { return codeDirectory; @@ -140,7 +112,6 @@ public void afterPropertiesSet() throws Exception { connection.setAutoCommit(false); if (codeDirectory != null && !codeDirectory.trim().equals("")) { logger.info("Loading vocabularies at: " + codeDirectory + "..."); -// loadDirectory(codeDirectory, connection); loadDirectory(codeDirectory, dataSource); logger.info("Vocabularies loaded..."); } @@ -148,7 +119,6 @@ public void afterPropertiesSet() throws Exception { if (valueSetDirectory != null && !valueSetDirectory.trim().equals("")) { logger.info("Loading value sets at: " + valueSetDirectory + "..."); -// loadDirectory(valueSetDirectory, connection); loadDirectory(valueSetDirectory, dataSource); logger.info("Value Sets loaded..."); } @@ -162,16 +132,6 @@ public void afterPropertiesSet() throws Exception { connection.commit(); logger.info("!!!!*********** INITIALIZED HASH SETS. ***********!!!!"); - -// entityManagerFactory.getCache().evictAll(); -// EntityManager em = entityManagerFactory.createEntityManager(); -// EntityTransaction tx = em.getTransaction(); -// tx.begin(); -// em.flush(); -// em.clear(); -// tx.commit(); -// -// logger.info("!!!!*********** EM Evicted All. ***********!!!!"); } catch (Exception e) { logger.error("Failed to load configured vocabulary directory.", e); diff --git a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java index 95a97e8..e0876de 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/VocabularyLoader.java @@ -10,6 +10,5 @@ * Created by Brian on 2/7/2016. */ public interface VocabularyLoader { -// void load(List file, Connection connection); long load(List file, DataSource ds); } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java index ef01088..1444d7a 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/CdtLoader.java @@ -4,9 +4,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; import java.util.List; import javax.sql.DataSource; @@ -19,9 +16,6 @@ import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.sitenv.vocabularies.loader.BaseCodeLoader; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; @@ -62,31 +56,17 @@ public long load(List filesToLoad, DataSource ds) { code = codeCell.getStringCellValue(); displayName = descriptionCell.getStringCellValue(); -// insertQueryBuilder.append("("); -// insertQueryBuilder.append("'"); -// insertQueryBuilder.append(code.toUpperCase().trim()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(displayName.replaceAll("'", "''").toUpperCase().trim()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("')"); - n++; buildCodeInsertQueryString(insertQueryBuilder, code.toUpperCase(), displayName.toUpperCase(), codeSystem, CodeSystemOIDs.CDT.codesystemOID()); t.update(insertQueryPrefix.toString()); insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); - -// t.update(insertQueryPrefix,code.toUpperCase().trim(),displayName.toUpperCase().trim(),file.getParentFile().getName(),CodeSystemOIDs.CDT.codesystemOID()); } } - //insertCode(insertQueryBuilder.toString(), connection); } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { try { if(workBook != null) { @@ -104,78 +84,6 @@ public long load(List filesToLoad, DataSource ds) { return n; } -// public void load(List filesToLoad, Connection connection) { -// StrBuilder insertQueryBuilder = null; -//// String insertQueryPrefix = "insert into CODES (ID, CODE, DISPLAYNAME, CODESYSTEM) values "; -// String insertQueryPrefix = "insert into CODES ( CODE, DISPLAYNAME, CODESYSTEM) values "; -// for (File file : filesToLoad) { -// if (file.isFile() && !file.isHidden()) { -// InputStream inputStream = null; -// XSSFWorkbook workBook = null; -// try { -// logger.debug("Loading CDT File: " + file.getName()); -// inputStream = new FileInputStream(file); -// workBook = new XSSFWorkbook(inputStream); -// -// for (int i = 0; i < workBook.getNumberOfSheets(); i++) { -// insertQueryBuilder = new StrBuilder(insertQueryPrefix); -// XSSFSheet sheet = workBook.getSheetAt(i); -// -// for (int count = 26; count < sheet.getLastRowNum()+1; count++) { -// if (!isRowEmpty(sheet.getRow(count))) { -// String code; -// String displayName; -// -// XSSFCell codeCell = sheet.getRow(count).getCell(0); -// XSSFCell descriptionCell = sheet.getRow(count).getCell(2); -// -// codeCell.setCellType(Cell.CELL_TYPE_STRING); -// descriptionCell.setCellType(Cell.CELL_TYPE_STRING); -// -// -// code = codeCell.getStringCellValue(); -// displayName = descriptionCell.getStringCellValue(); -// -// if(count > 26){ -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -//// insertQueryBuilder.append("DEFAULT"); -//// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append("'"); -// insertQueryBuilder.append(code.toUpperCase().trim()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(displayName.replaceAll("'", "''").toUpperCase().trim()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("')"); -// -//// CcdaCodeSystemCodeValidatorDAO.addRow(code.toUpperCase().trim(),file.getParentFile().getName(),displayName.toUpperCase().trim()); -// -// } -// } -// doInsert(insertQueryBuilder.toString(), connection); -// } -// } catch (IOException e) { -// logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); -// } finally { -// try { -// if(workBook != null) { -// workBook.close(); -// } -// if(inputStream != null) { -// inputStream.close(); -// } -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// } -// } -// } - public static boolean isRowEmpty(Row row) { if(row != null){ for (int c = row.getFirstCellNum(); c < row.getLastCellNum(); c++) { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java index a23eb18..627d2bd 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/CptLoader.java @@ -1,20 +1,18 @@ package org.sitenv.vocabularies.loader.code; -import org.apache.commons.lang3.text.StrBuilder; -import org.apache.log4j.Logger; -import org.sitenv.vocabularies.loader.BaseCodeLoader; -import org.springframework.stereotype.Component; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.List; import javax.sql.DataSource; + +import org.apache.commons.lang3.text.StrBuilder; +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.loader.BaseCodeLoader; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; /** * Created by Brian on 2/7/2016. @@ -35,7 +33,6 @@ public long load(List filesToLoad, DataSource ds) { FileReader fileReader = null; try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); - int totalCount = 0, pendingCount = 0; JdbcTemplate t = new JdbcTemplate(ds); for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { @@ -50,13 +47,6 @@ public long load(List filesToLoad, DataSource ds) { String displayName = line.substring(line.indexOf(" ")); buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, oid); -// if ((++totalCount % BATCH_SIZE) == 0) { -// insertCode(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(codeTableInsertSQLPrefix); -// pendingCount = 0; -// } - n++; t.update(codeTableInsertSQLPrefix,code.toUpperCase().trim(),displayName.toUpperCase().trim(),codeSystem,CodeSystemOIDs.CDT.codesystemOID()); @@ -64,15 +54,9 @@ public long load(List filesToLoad, DataSource ds) { } } } -// if (pendingCount > 0) { -// insertCode(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); } -// catch (SQLException e) { -// e.printStackTrace(); -// } finally { if (br != null) { try { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java index 39915c7..1efbcc5 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/Icd10BaseLoader.java @@ -42,26 +42,8 @@ public long load(List filesToLoad, DataSource datasource) { String code = buildDelimitedIcdCode(available.substring(6, 13)); String shortDisplayName = available.substring(16, 77); String longDisplayName = available.substring(77); -// if (pendingCount++ > 0) { -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -// insertQueryBuilder.append("DEFAULT"); -// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append(buildDelimitedIcdCode(available.substring(6, 13).trim()).toUpperCase()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(available.substring(77).trim().toUpperCase().replaceAll("'", "''")); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(oid); -// insertQueryBuilder.append("')"); n++; - - //@TODO Merge to update statements into single update statement ????? - //@TODO Merge to update statements into single update statement ????? - buildCodeInsertQueryString(insertQueryBuilder, code, shortDisplayName, codeSystem, oid); t.update(insertQueryBuilder.toString()); @@ -73,28 +55,12 @@ public long load(List filesToLoad, DataSource datasource) { insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); -// t.update(codeTableInsertSQLPrefix,buildDelimitedIcdCode(code.trim()).toUpperCase(),longDisplayName.trim().toUpperCase(),file.getParentFile().getName(),oid); -// t.update(codeTableInsertSQLPrefix,buildDelimitedIcdCode(code.trim()).toUpperCase(),shortDisplayName.trim().toUpperCase(),file.getParentFile().getName(),oid); - -// t.update(insertQueryPrefix,buildDelimitedIcdCode(available.substring(6, 13).trim()).toUpperCase(),available.substring(77).trim().toUpperCase(),file.getParentFile().getName(),oid); - -// if ((++totalCount % 2500) == 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(insertQueryPrefix); -// pendingCount = 0; -// } } } } -// if (pendingCount > 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { if (br != null) { try { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java index a268040..9278982 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/Icd9BaseLoader.java @@ -1,22 +1,19 @@ package org.sitenv.vocabularies.loader.code; -import org.apache.commons.lang3.text.StrBuilder; -import org.apache.log4j.Logger; -import org.sitenv.vocabularies.loader.BaseCodeLoader; -import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; -import org.springframework.jdbc.core.JdbcTemplate; +import static org.sitenv.vocabularies.loader.code.IcdLoader.buildDelimitedIcdCode; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.List; import javax.sql.DataSource; -import static org.sitenv.vocabularies.loader.code.IcdLoader.buildDelimitedIcdCode; +import org.apache.commons.lang3.text.StrBuilder; +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.loader.BaseCodeLoader; +import org.springframework.jdbc.core.JdbcTemplate; /** * Created by Brian on 2/7/2016. @@ -33,7 +30,6 @@ public long load(List filesToLoad, DataSource datasource) { JdbcTemplate t = new JdbcTemplate(datasource); try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); - int totalCount = 0, pendingCount = 0; for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { @@ -47,30 +43,17 @@ public long load(List filesToLoad, DataSource datasource) { String code = buildDelimitedIcdCode(line.substring(0, 5)); String displayName = line.substring(6); buildCodeInsertQueryString(insertQueryBuilder, code, displayName, codeSystem, oid); -// -// if ((++totalCount % BATCH_SIZE) == 0) { -// insertCode(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(codeTableInsertSQLPrefix); -// pendingCount = 0; -// } n++; t.update(insertQueryBuilder.toString()); insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); -// t.update(codeTableInsertSQLPrefix,code.trim().toUpperCase(),displayName.trim().toUpperCase(),codeSystem,oid); } } } } -// if (pendingCount > 0) { -// insertCode(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { if (br != null) { try { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java index 6ebd623..bfbc4e4 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/IcdLoader.java @@ -44,6 +44,4 @@ protected static String buildDelimitedIcdCode(String code) { return builder.build(); } - -// protected abstract void setOID(String oid); } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java index 7fb6d48..9d0273c 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/LoincLoader.java @@ -1,24 +1,21 @@ package org.sitenv.vocabularies.loader.code; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.text.StrBuilder; -import org.apache.log4j.Logger; -import org.sitenv.vocabularies.loader.BaseCodeLoader; -import org.sitenv.vocabularies.loader.VocabularyLoader; -import org.sitenv.vocabularies.validation.dao.CodeSystemCodeDAO; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Component; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.List; import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.text.StrBuilder; +import org.apache.log4j.Logger; +import org.sitenv.vocabularies.loader.BaseCodeLoader; +import org.sitenv.vocabularies.loader.VocabularyLoader; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + /** * Created by Brian on 2/7/2016. */ @@ -33,7 +30,6 @@ public long load(List filesToLoad, DataSource datasource) { FileReader fileReader = null; try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); - int totalCount = 0, pendingCount = 0; for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { @@ -47,20 +43,6 @@ public long load(List filesToLoad, DataSource datasource) { continue; // skip header row } else { String[] line = available.replaceAll("^\"", "").split("\"?(,|$)(?=(([^\"]*\"){2})*[^\"]*$) *\"?"); -// if (pendingCount++ > 0) { -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -// insertQueryBuilder.append("DEFAULT"); -// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append(StringUtils.strip(line[0], "\"").toUpperCase()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(StringUtils.strip(line[29], "\"").toUpperCase().replaceAll("'", "''")); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(CodeSystemOIDs.LOINC.codesystemOID()); -// insertQueryBuilder.append("')"); String code = StringUtils.strip(line[0], "\""); String codeSystem = file.getParentFile().getName(); @@ -85,31 +67,12 @@ public long load(List filesToLoad, DataSource datasource) { t.update(insertQueryBuilder.toString()); insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); - - -// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),longCommonName.toUpperCase(),codeSystem,oid); -// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),componentName.toUpperCase(),codeSystem,oid); -// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),shortName.toUpperCase(),codeSystem,oid); - -// t.update(insertQueryPrefix,StringUtils.strip(line[0], "\"").toUpperCase(),StringUtils.strip(line[29], "\"").toUpperCase(),file.getParentFile().getName(),CodeSystemOIDs.LOINC.codesystemOID()); - -// if ((++totalCount % 5000) == 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(insertQueryPrefix); -// pendingCount = 0; -// } } } } } -// if (pendingCount > 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { if (br != null) { try { @@ -122,73 +85,4 @@ public long load(List filesToLoad, DataSource datasource) { } return n; } - -// @Override -// public void load(List filesToLoad, Connection connection) { -// BufferedReader br = null; -// FileReader fileReader = null; -// try { -//// String insertQueryPrefix = "insert into CODES (ID, CODE, DISPLAYNAME, CODESYSTEM) values "; -// String insertQueryPrefix = "insert into CODES ( CODE, DISPLAYNAME, CODESYSTEM) values "; -// StrBuilder insertQueryBuilder = new StrBuilder(insertQueryPrefix); -// int totalCount = 0, pendingCount = 0; -// -// for (File file : filesToLoad) { -// if (file.isFile() && !file.isHidden()) { -// logger.debug("Loading LOINC File: " + file.getName()); -// int count = 0; -// fileReader = new FileReader(file); -// br = new BufferedReader(fileReader); -// String available; -// while ((available = br.readLine()) != null) { -// if ((count++ == 0)) { -// continue; // skip header row -// } else { -// String[] line = StringUtils.splitPreserveAllTokens(available, ",", 3); -// if (pendingCount++ > 0) { -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -//// insertQueryBuilder.append("DEFAULT"); -//// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append("'"); -// insertQueryBuilder.append(StringUtils.strip(line[0], "\"").toUpperCase()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(StringUtils.strip(line[1], "\"").toUpperCase().replaceAll("'", "''")); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("')"); -// -//// CcdaCodeSystemCodeValidatorDAO.addRow(StringUtils.strip(line[0], "\"").toUpperCase(), file.getParentFile().getName(), -//// StringUtils.strip(line[1], "\"").toUpperCase()); -// -// if ((++totalCount % 900) == 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(insertQueryPrefix); -// pendingCount = 0; -// } -// } -// } -// } -// } -// if (pendingCount > 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// } -// } catch (IOException e) { -// logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); -// } finally { -// if (br != null) { -// try { -// fileReader.close(); -// br.close(); -// } catch (IOException e) { -// logger.error(e); -// } -// } -// } -// } } diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java index c218e3d..f50ecf7 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/RxNormLoader.java @@ -30,7 +30,6 @@ public long load(List filesToLoad, DataSource datasource) { BufferedReader br = null; try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); - int totalCount = 0, pendingCount = 0; for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { @@ -43,20 +42,6 @@ public long load(List filesToLoad, DataSource datasource) { String[] line = StringUtils.splitPreserveAllTokens(available, "|", 16); String code = line[0]; String displayName = line[14]; -// if (pendingCount++ > 0) { -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -// insertQueryBuilder.append("DEFAULT"); -// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append(line[0]); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(line[14].toUpperCase().replaceAll("'", "''")); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(CodeSystemOIDs.RXNORM.codesystemOID()); -// insertQueryBuilder.append("')"); n++; buildCodeInsertQueryString(insertQueryBuilder, code, displayName.toUpperCase(), codeSystem, CodeSystemOIDs.RXNORM.codesystemOID()); @@ -64,24 +49,11 @@ public long load(List filesToLoad, DataSource datasource) { insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); -// t.update(codeTableInsertSQLPrefix,code,displayName.toUpperCase(),codeSystem,CodeSystemOIDs.RXNORM.codesystemOID()); - -// if ((++totalCount % 5000) == 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(insertQueryPrefix); -// pendingCount = 0; -// } } } } -// if (pendingCount > 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { if (br != null) { try { diff --git a/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java b/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java index 9867bff..64966e8 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/code/SnomedLoader.java @@ -30,7 +30,6 @@ public long load(List filesToLoad, DataSource datasource) { FileReader fileReader = null; try { StrBuilder insertQueryBuilder = new StrBuilder(codeTableInsertSQLPrefix); - int totalCount = 0, pendingCount = 0; for (File file : filesToLoad) { if (file.isFile() && !file.isHidden()) { @@ -47,49 +46,17 @@ public long load(List filesToLoad, DataSource datasource) { String[] line = StringUtils.splitPreserveAllTokens(available, "\t", 9); String code = line[4]; String displayName = line[7]; -// if (pendingCount++ > 0) { -// insertQueryBuilder.append(","); -// } -// insertQueryBuilder.append("("); -// insertQueryBuilder.append("DEFAULT"); -// insertQueryBuilder.append(",'"); -// insertQueryBuilder.append(line[4].toUpperCase()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(line[7].toUpperCase().replaceAll("'", "''")); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(file.getParentFile().getName()); -// insertQueryBuilder.append("','"); -// insertQueryBuilder.append(CodeSystemOIDs.SNOMEDCT.codesystemOID()); -// insertQueryBuilder.append("')"); - n++; - - buildCodeInsertQueryString(insertQueryBuilder, code.toUpperCase(), displayName.toUpperCase(), codeSystem, CodeSystemOIDs.SNOMEDCT.codesystemOID()); t.update(insertQueryBuilder.toString()); insertQueryBuilder.clear(); insertQueryBuilder.append(codeTableInsertSQLPrefix); - - -// t.update(codeTableInsertSQLPrefix,code.toUpperCase(),displayName.toUpperCase(),codeSystem,CodeSystemOIDs.SNOMEDCT.codesystemOID()); -// t.update(insertQueryPrefix,line[4].toUpperCase(),line[7].toUpperCase(),file.getParentFile().getName(),CodeSystemOIDs.SNOMEDCT.codesystemOID()); -// if ((++totalCount % 5000) == 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// insertQueryBuilder.clear(); -// insertQueryBuilder.append(insertQueryPrefix); -// pendingCount = 0; -// } } } } } -// if (pendingCount > 0) { -// doInsert(insertQueryBuilder.toString(), connection); -// } } catch (IOException e) { logger.error(e); -// } catch (SQLException e) { -// e.printStackTrace(); } finally { if (br != null) { try { diff --git a/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java b/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java index b9e5952..85f8494 100644 --- a/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java +++ b/src/main/java/org/sitenv/vocabularies/loader/valueset/VsacLoader.java @@ -27,7 +27,6 @@ public class VsacLoader extends BaseCodeLoader implements VocabularyLoader { private static Logger logger = Logger.getLogger(VsacLoader.class); private static final int MIN_EXPECTED_NUMBER_OF_CELLS_IN_ROW = 6; private static final int CODE_CELL_INDEX_IN_ROW = 0; - private static final int BATCH_SIZE = 1000; private static final String HEADER_ROW_FINDER_KEY = "CODE"; public long load(List filesToLoad, DataSource datasource) { @@ -38,9 +37,7 @@ public long load(List filesToLoad, DataSource datasource) { if (file.isFile() && !file.isHidden()) { try { logger.info("Loading Value Set File: " + file.getName()); -// PreparedStatement preparedStatement = connection.prepareStatement(insertQueryPrefix); InputStream inputStream = new FileInputStream(file); -// Workbook workBook = StreamingReader.builder().open(inputStream); Workbook workBook = WorkbookFactory.create(inputStream); for (int i = 1; i < workBook.getNumberOfSheets(); i++) { boolean headerRowFound = false; @@ -50,7 +47,6 @@ public long load(List filesToLoad, DataSource datasource) { String valueSetType = ""; String valueSetVersion = ""; String valueSetSteward = ""; - int valuesetDataRowCount = 0; for(Row row : sheet){ if(row.getRowNum() < 6) { @@ -79,20 +75,7 @@ public long load(List filesToLoad, DataSource datasource) { if(row.getRowNum() > 10){ if(row.getCell(0) != null) { row.getCell(0).setCellType(Cell.CELL_TYPE_STRING); -// if (!StringUtils.isEmpty(row.getCell(0).getStringCellValue())) { if(headerRowFound && canProcessRow(row)){ -// preparedStatement.setString(1, row.getCell(0).getStringCellValue().replaceAll("'", "''").toUpperCase().trim()); -// preparedStatement.setString(2, row.getCell(1).getStringCellValue().replaceAll("'", "''").toUpperCase().trim()); -// preparedStatement.setString(3, row.getCell(2).getStringCellValue().toUpperCase().trim()); -// preparedStatement.setString(4, row.getCell(3).getStringCellValue().trim()); -// preparedStatement.setString(5, row.getCell(4).getStringCellValue().toUpperCase().trim()); -// preparedStatement.setString(6, row.getCell(5).getStringCellValue().toUpperCase().trim()); -// preparedStatement.setString(7, valueSetName); -// preparedStatement.setString(8, valueSetOid); -// preparedStatement.setString(9, valueSetType); -// preparedStatement.setString(10, valueSetVersion); -// preparedStatement.setString(11, valueSetSteward); -// preparedStatement.addBatch(); n++; row.getCell(1).setCellType(Cell.CELL_TYPE_STRING); row.getCell(2).setCellType(Cell.CELL_TYPE_STRING); @@ -112,28 +95,13 @@ public long load(List filesToLoad, DataSource datasource) { valueSetSteward); } } -// if(row.getRowNum() % 1000 == 0){ -// preparedStatement.executeBatch(); -// connection.commit(); -// preparedStatement.clearBatch(); -// valuesetDataRowCount++; - -// if(valuesetDataRowCount % BATCH_SIZE == 0){ -// preparedStatement.executeBatch(); -// connection.commit(); -// preparedStatement.clearBatch(); -// } -// } } if(!headerRowFound){ if(hasValueInCell(row, 0) && row.getCell(0).getStringCellValue().toUpperCase().trim().equals(HEADER_ROW_FINDER_KEY)){ headerRowFound = true; } -// preparedStatement.executeBatch(); -// connection.commit(); } - } } workBook.close(); diff --git a/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java b/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java index 5fcd8e2..3cf86bc 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java +++ b/src/main/java/org/sitenv/vocabularies/validation/NodeValidation.java @@ -1,15 +1,15 @@ package org.sitenv.vocabularies.validation; +import java.util.List; + +import javax.xml.xpath.XPath; + import org.sitenv.vocabularies.configuration.ConfiguredValidator; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; -import org.w3c.dom.Node; import com.ximpleware.VTDNav; -import javax.xml.xpath.XPath; -import java.util.List; - public interface NodeValidation { List validateNode(ConfiguredValidator configuredValidator, XPath xpath, VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression); } diff --git a/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java index 29b8b2d..3cb05ff 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java +++ b/src/main/java/org/sitenv/vocabularies/validation/dao/CodeSystemCodeDAO.java @@ -65,11 +65,6 @@ public synchronized void loadCodes(DataSource ds) { if (done) { return; } - // String sql = "SELECT codeSystem FROM Codes"; - // NamedParameterJdbcTemplate t = new - // NamedParameterJdbcTemplate(datasource); - // Map paramMap = new HashMap(); - // codes = t.queryForList(sql, paramMap, String.class); String sql = "SELECT code, displayName,codeSystem, codeSystemOID FROM Codes"; JdbcTemplate tmpl = new JdbcTemplate(ds); diff --git a/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java b/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java index ac0e761..88a0f7f 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java +++ b/src/main/java/org/sitenv/vocabularies/validation/dao/ValueSetDAO.java @@ -2,10 +2,8 @@ import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -13,7 +11,6 @@ import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.sitenv.vocabularies.validation.entities.VsacValueSet; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowCallbackHandler; import org.springframework.stereotype.Component; diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java index ad7c603..7ad6a85 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java +++ b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateRequest.java @@ -3,7 +3,6 @@ import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredExpression; import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionFilter; -import org.w3c.dom.Document; import com.ximpleware.VTDNav; diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java index b3f71a1..69c47e6 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java +++ b/src/main/java/org/sitenv/vocabularies/validation/services/ValidateWorker.java @@ -36,10 +36,8 @@ @Component public class ValidateWorker { - private XPathFactory xPathFactory; - public ValidateWorker() { - this.xPathFactory = XPathFactory.newInstance(); + XPathFactory.newInstance(); } @Resource(name = "vocabularyValidationConfigurations") diff --git a/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java b/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java index 85f7b20..d03c765 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java +++ b/src/main/java/org/sitenv/vocabularies/validation/services/VocabularyValidationService.java @@ -1,6 +1,5 @@ package org.sitenv.vocabularies.validation.services; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -15,11 +14,6 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; @@ -28,18 +22,13 @@ import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredExpression; -import org.sitenv.vocabularies.configuration.ConfiguredValidator; -import org.sitenv.vocabularies.validation.NodeValidation; import org.sitenv.vocabularies.validation.NodeValidatorFactory; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; -import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; import org.sitenv.vocabularies.validation.utils.CCDADocumentNamespaces; import org.sitenv.vocabularies.validation.utils.ConfiguredExpressionLevelFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import com.ximpleware.VTDGen; @@ -91,9 +80,6 @@ private DocumentBuilder getDocumentBuilder() throws ParserConfigurationException @Resource(name = "xPathFactory") XPathFactory xPathFactory; - // @Autowired - // VocabularyValidatorFactory vocabularyValidatorFactory; - // ------------------------- INTERNAL CODE CHANGE START // -------------------------- public List validate(String uri, String severityLevel) diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java index 3af676f..49eb6e8 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ClassCodeValidator.java @@ -23,15 +23,9 @@ @Component(value = "ClassCodeValidator") public class ClassCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ClassCodeValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; - -// @Autowired -// public ClassCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java index f066749..c179063 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/CodeSystemCodeValidator.java @@ -25,16 +25,10 @@ @Component(value = "CodeSystemCodeValidator") public class CodeSystemCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(CodeSystemCodeValidator.class); -// private CodeRepository codeRepository; @Autowired CodeSystemCodeDAO codeRepository; -// @Autowired -// public CodeSystemCodeValidator(CodeRepository codeRepository) { -// this.codeRepository = codeRepository; -// } - @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java index bdbac0f..4d80580 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeCountryCodeValuesetValidator.java @@ -25,16 +25,10 @@ @Component(value = "LanguageCodeNodeCountryCodeValuesetValidator") public class LanguageCodeNodeCountryCodeValuesetValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(LanguageCodeNodeCountryCodeValuesetValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; -// @Autowired -// public LanguageCodeNodeCountryCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } - @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, VTDNav vn, int nodeIndex, ConfiguredExpressionFilter filter, String xpathExpression) { diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java index 0191e7e..889ffc5 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/LanguageCodeNodeLanguageCodeValuesetValidator.java @@ -26,15 +26,10 @@ @Component(value = "LanguageCodeNodeLanguageCodeValuesetValidator") public class LanguageCodeNodeLanguageCodeValuesetValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(LanguageCodeNodeLanguageCodeValuesetValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; -// @Autowired -// public LanguageCodeNodeLanguageCodeValuesetValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java index 23926d7..840ba5e 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/NodeCodeSystemMatchesConfiguredCodeSystemValidator.java @@ -9,7 +9,6 @@ import org.apache.log4j.Logger; import org.sitenv.vocabularies.configuration.ConfiguredValidationResultSeverityLevel; import org.sitenv.vocabularies.configuration.ConfiguredValidator; -import org.sitenv.vocabularies.validation.NodeValidation; import org.sitenv.vocabularies.validation.dto.NodeValidationResult; import org.sitenv.vocabularies.validation.dto.VocabularyValidationResult; import org.sitenv.vocabularies.validation.dto.enums.VocabularyValidationResultLevel; @@ -17,8 +16,6 @@ import org.sitenv.vocabularies.validation.utils.XpathUtils; import org.sitenv.vocabularies.validation.validators.NodeValidator; import org.springframework.stereotype.Component; -import org.w3c.dom.Element; -import org.w3c.dom.Node; import com.ximpleware.VTDNav; diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java index fbfce29..d64d701 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/TextNodeValidator.java @@ -24,15 +24,10 @@ @Component(value = "TextNodeValidator") public class TextNodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(TextNodeValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; -// @Autowired -// public TextNodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java index 886bd61..12b6465 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/UnitValidator.java @@ -24,15 +24,10 @@ @Component(value = "UnitValidator") public class UnitValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(UnitValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; -// @Autowired -// public UnitValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java index 8453d0f..6ece623 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetCodeValidator.java @@ -24,15 +24,10 @@ @Component(value = "ValueSetCodeValidator") public class ValueSetCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ValueSetCodeValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; -// @Autowired -// public ValueSetCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath, diff --git a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java index 02c0771..d4fca6c 100644 --- a/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java +++ b/src/main/java/org/sitenv/vocabularies/validation/validators/nodetypes/ValueSetNodeWithOnlyCodeValidator.java @@ -24,15 +24,9 @@ @Component(value = "ValueSetNodeWithOnlyCodeValidator") public class ValueSetNodeWithOnlyCodeValidator extends NodeValidator { private static final Logger logger = Logger.getLogger(ValueSetNodeWithOnlyCodeValidator.class); -// private VsacValuesSetRepository vsacValuesSetRepository; @Autowired ValueSetDAO vsacValuesSetRepository; - -// @Autowired -// public ValueSetNodeWithOnlyCodeValidator(VsacValuesSetRepository vsacValuesSetRepository) { -// this.vsacValuesSetRepository = vsacValuesSetRepository; -// } @Override public List validateNode(ConfiguredValidator configuredValidator, XPath xpath,