diff --git a/README.md b/README.md index 908b1de..9a04172 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,20 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri * `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort * `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss. +## Transformation von Werten + +In Onkostar kann es vorkommen, dass ein Wert eines Merkmalskatalogs an einem Standort angepasst wurde und dadurch nicht dem Wert entspricht, +der vom bwHC-Backend akzeptiert wird. + +Diese Anwendung bietet daher die Möglichkeit, eine Transformation vorzunehmen. Hierzu muss der "Pfad" innerhalb des JSON-MTB-Files angegeben werden und +welcher Wert wie ersetzt werden soll. + +Hier ein Beispiel für die erste (Index 0 - weitere dann mit 1,2,...) Transformationsregel: + +* `APP_TRANSFORMATIONS_[0]_PATH`: Pfad zum Wert in der JSON-MTB-Datei. Beispiel: `diagnoses[*].icd10.version` für **alle** Diagnosen +* `APP_TRANSFORMATIONS_[0]_FROM`: Angabe des Werts, der ersetzt werden soll. Andere Werte bleiben dabei unverändert. +* `APP_TRANSFORMATIONS_[0]_TO`: Angabe des neuen Werts. + ## Mögliche Endpunkte Für REST-Requests als auch zur Nutzung von Kafka-Topics können Endpunkte konfiguriert werden. diff --git a/build.gradle.kts b/build.gradle.kts index 3f053f5..0ed1f37 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -69,6 +69,7 @@ dependencies { implementation("ca.uhn.hapi.fhir:hapi-fhir-base:${versions["hapi-fhir"]}") implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}") implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}") + implementation("com.jayway.jsonpath:json-path") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") runtimeOnly("org.postgresql:postgresql") developmentOnly("org.springframework.boot:spring-boot-devtools") diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt index 8bdaa60..99a5c72 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt @@ -31,13 +31,13 @@ import org.springframework.beans.factory.NoSuchBeanDefinitionException import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.mock.mockito.MockBean -import org.springframework.boot.test.mock.mockito.MockBeans import org.springframework.context.ApplicationContext import org.springframework.test.context.ContextConfiguration import org.springframework.test.context.TestPropertySource @SpringBootTest -@ContextConfiguration(classes = [KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class]) +@ContextConfiguration(classes = [AppConfiguration::class, KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class]) +@MockBean(ObjectMapper::class) class AppConfigurationTest { @Nested @@ -65,10 +65,7 @@ class AppConfigurationTest { "app.kafka.group-id=test" ] ) - @MockBeans(value = [ - MockBean(ObjectMapper::class), - MockBean(RequestRepository::class) - ]) + @MockBean(RequestRepository::class) inner class AppConfigurationKafkaTest(private val context: ApplicationContext) { @Test @@ -99,4 +96,24 @@ class AppConfigurationTest { } + @Nested + @TestPropertySource( + properties = [ + "app.transformations[0].path=consent.status", + "app.transformations[0].from=rejected", + "app.transformations[0].to=accept", + ] + ) + inner class AppConfigurationTransformationTest(private val context: ApplicationContext) { + + @Test + fun shouldRecognizeTransformations() { + val appConfigProperties = context.getBean(AppConfigProperties::class.java) + + assertThat(appConfigProperties).isNotNull + assertThat(appConfigProperties.transformations).hasSize(1) + } + + } + } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 06e730b..6b85603 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -24,7 +24,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(AppConfigProperties.NAME) data class AppConfigProperties( var bwhcUri: String?, - var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN + var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN, + var transformations: List = listOf() ) { companion object { const val NAME = "app" @@ -78,4 +79,10 @@ data class KafkaTargetProperties( enum class PseudonymGenerator { BUILDIN, GPAS -} \ No newline at end of file +} + +data class TransformationProperties( + val path: String, + val from: String, + val to: String +) \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 6b15fc0..c8e86fb 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -25,6 +25,9 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import dev.dnpm.etl.processor.services.Transformation +import dev.dnpm.etl.processor.services.TransformationService +import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -41,6 +44,8 @@ import reactor.core.publisher.Sinks ) class AppConfiguration { + private val logger = LoggerFactory.getLogger(AppConfiguration::class.java) + @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS") @Bean fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator { @@ -71,5 +76,16 @@ class AppConfiguration { return Sinks.many().multicast().directBestEffort() } + @Bean + fun transformationService( + objectMapper: ObjectMapper, + configProperties: AppConfigProperties + ): TransformationService { + logger.info("Apply ${configProperties.transformations.size} transformation rules") + return TransformationService(objectMapper, configProperties.transformations.map { + Transformation.of(it.path) from it.from to it.to + }) + } + } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 3cd912c..fd9a3f5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -38,6 +38,7 @@ import java.util.* @Service class RequestProcessor( private val pseudonymizeService: PseudonymizeService, + private val transformationService: TransformationService, private val sender: MtbFileSender, private val requestService: RequestService, private val objectMapper: ObjectMapper, @@ -50,7 +51,7 @@ class RequestProcessor( mtbFile pseudonymizeWith pseudonymizeService - val request = MtbFileSender.MtbFileRequest(requestId, mtbFile) + val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile)) requestService.save( Request( diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt new file mode 100644 index 0000000..26de550 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt @@ -0,0 +1,81 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.jayway.jsonpath.JsonPath +import com.jayway.jsonpath.PathNotFoundException +import de.ukw.ccc.bwhc.dto.MtbFile + +class TransformationService(private val objectMapper: ObjectMapper, private val transformations: List) { + fun transform(mtbFile: MtbFile): MtbFile { + var json = objectMapper.writeValueAsString(mtbFile) + + transformations.forEach { transformation -> + val jsonPath = JsonPath.parse(json) + + try { + val before = transformation.path.substringBeforeLast(".") + val last = transformation.path.substringAfterLast(".") + + val existingValue = if (transformation.existingValue is Number) transformation.existingValue else transformation.existingValue.toString() + val newValue = if (transformation.newValue is Number) transformation.newValue else transformation.newValue.toString() + + jsonPath.set("$.$before.[?]$last", newValue, { + it.item(HashMap::class.java)[last] == existingValue + }) + } catch (e: PathNotFoundException) { + // Ignore + } + + json = jsonPath.jsonString() + } + + return objectMapper.readValue(json, MtbFile::class.java) + } + +} + +class Transformation private constructor(internal val path: String) { + + lateinit var existingValue: Any + private set + lateinit var newValue: Any + private set + + infix fun from(value: Any): Transformation { + this.existingValue = value + return this + } + + infix fun to(value: Any): Transformation { + this.newValue = value + return this + } + + companion object { + + fun of(path: String): Transformation { + return Transformation(path) + } + + } + +} diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 7856833..9aaa091 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -37,6 +37,7 @@ import org.mockito.Mockito.* import org.mockito.junit.jupiter.MockitoExtension import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.whenever import org.springframework.context.ApplicationEventPublisher import java.time.Instant import java.util.* @@ -46,6 +47,7 @@ import java.util.* class RequestProcessorTest { private lateinit var pseudonymizeService: PseudonymizeService + private lateinit var transformationService: TransformationService private lateinit var sender: MtbFileSender private lateinit var requestService: RequestService private lateinit var applicationEventPublisher: ApplicationEventPublisher @@ -55,11 +57,13 @@ class RequestProcessorTest { @BeforeEach fun setup( @Mock pseudonymizeService: PseudonymizeService, + @Mock transformationService: TransformationService, @Mock sender: RestMtbFileSender, @Mock requestService: RequestService, @Mock applicationEventPublisher: ApplicationEventPublisher ) { this.pseudonymizeService = pseudonymizeService + this.transformationService = transformationService this.sender = sender this.requestService = requestService this.applicationEventPublisher = applicationEventPublisher @@ -68,6 +72,7 @@ class RequestProcessorTest { requestProcessor = RequestProcessor( pseudonymizeService, + transformationService, sender, requestService, objectMapper, @@ -98,6 +103,10 @@ class RequestProcessorTest { it.arguments[0] as String }.`when`(pseudonymizeService).patientPseudonym(any()) + doAnswer { + it.arguments[0] + }.whenever(transformationService).transform(any()) + val mtbFile = MtbFile.builder() .withPatient( Patient.builder() @@ -153,6 +162,10 @@ class RequestProcessorTest { it.arguments[0] as String }.`when`(pseudonymizeService).patientPseudonym(any()) + doAnswer { + it.arguments[0] + }.whenever(transformationService).transform(any()) + val mtbFile = MtbFile.builder() .withPatient( Patient.builder() @@ -212,6 +225,10 @@ class RequestProcessorTest { it.arguments[0] as String }.`when`(pseudonymizeService).patientPseudonym(any()) + doAnswer { + it.arguments[0] + }.whenever(transformationService).transform(any()) + val mtbFile = MtbFile.builder() .withPatient( Patient.builder() @@ -271,6 +288,10 @@ class RequestProcessorTest { it.arguments[0] as String }.`when`(pseudonymizeService).patientPseudonym(any()) + doAnswer { + it.arguments[0] + }.whenever(transformationService).transform(any()) + val mtbFile = MtbFile.builder() .withPatient( Patient.builder() diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/TransformationServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/TransformationServiceTest.kt new file mode 100644 index 0000000..487b502 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/TransformationServiceTest.kt @@ -0,0 +1,95 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import de.ukw.ccc.bwhc.dto.Consent +import de.ukw.ccc.bwhc.dto.Diagnosis +import de.ukw.ccc.bwhc.dto.Icd10 +import de.ukw.ccc.bwhc.dto.MtbFile +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class TransformationServiceTest { + + private lateinit var service: TransformationService + + @BeforeEach + fun setup() { + this.service = TransformationService( + ObjectMapper(), listOf( + Transformation.of("consent.status") from Consent.Status.ACTIVE to Consent.Status.REJECTED, + Transformation.of("diagnoses[*].icd10.version") from "2013" to "2014", + ) + ) + } + + @Test + fun shouldTransformMtbFile() { + val mtbFile = MtbFile.builder().withDiagnoses( + listOf( + Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also { + it.version = "2013" + }).build() + ) + ).build() + + val actual = this.service.transform(mtbFile) + + assertThat(actual).isNotNull + assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014") + } + + @Test + fun shouldOnlyTransformGivenValues() { + val mtbFile = MtbFile.builder().withDiagnoses( + listOf( + Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also { + it.version = "2013" + }).build(), + Diagnosis.builder().withId("5678").withIcd10(Icd10("F79.8").also { + it.version = "2019" + }).build() + ) + ).build() + + val actual = this.service.transform(mtbFile) + + assertThat(actual).isNotNull + assertThat(actual.diagnoses[0].icd10.code).isEqualTo("F79.9") + assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014") + assertThat(actual.diagnoses[1].icd10.code).isEqualTo("F79.8") + assertThat(actual.diagnoses[1].icd10.version).isEqualTo("2019") + } + + @Test + fun shouldTransformMtbFileWithConsentEnum() { + val mtbFile = MtbFile.builder().withConsent( + Consent("123", "456", Consent.Status.ACTIVE) + ).build() + + val actual = this.service.transform(mtbFile) + + assertThat(actual.consent).isNotNull + assertThat(actual.consent.status).isEqualTo(Consent.Status.REJECTED) + } + +} \ No newline at end of file