pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00
Code Issues Releases Activity

Merge pull request #13 from CCC-MF/issue_12

Browse Source 
Transformation of MTBFile data based on rules
This commit is contained in:
Paul-Christian Volkmer 2023-10-05 12:41:49 +02:00 committed by GitHub
commit 0eee1908df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 262 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -49,6 +49,20 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri
* `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort
* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
## Transformation von Werten
In Onkostar kann es vorkommen, dass ein Wert eines Merkmalskatalogs an einem Standort angepasst wurde und dadurch nicht dem Wert entspricht,
der vom bwHC-Backend akzeptiert wird.
Diese Anwendung bietet daher die Möglichkeit, eine Transformation vorzunehmen. Hierzu muss der "Pfad" innerhalb des JSON-MTB-Files angegeben werden und
welcher Wert wie ersetzt werden soll.
Hier ein Beispiel für die erste (Index 0 - weitere dann mit 1,2,...) Transformationsregel:
* `APP_TRANSFORMATIONS_[0]_PATH`: Pfad zum Wert in der JSON-MTB-Datei. Beispiel: `diagnoses[*].icd10.version` für **alle** Diagnosen
* `APP_TRANSFORMATIONS_[0]_FROM`: Angabe des Werts, der ersetzt werden soll. Andere Werte bleiben dabei unverändert.
* `APP_TRANSFORMATIONS_[0]_TO`: Angabe des neuen Werts.
## Mögliche Endpunkte
Für REST-Requests als auch zur Nutzung von Kafka-Topics können Endpunkte konfiguriert werden.

1
build.gradle.kts
View File

@ -69,6 +69,7 @@ dependencies {
implementation("ca.uhn.hapi.fhir:hapi-fhir-base:${versions["hapi-fhir"]}")
implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}")
implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}")
implementation("com.jayway.jsonpath:json-path")
runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
runtimeOnly("org.postgresql:postgresql")
developmentOnly("org.springframework.boot:spring-boot-devtools")

29
src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt
View File

@ -31,13 +31,13 @@ import org.springframework.beans.factory.NoSuchBeanDefinitionException
import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.boot.test.mock.mockito.MockBeans
import org.springframework.context.ApplicationContext
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
@SpringBootTest
@ContextConfiguration(classes = [KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class])
@ContextConfiguration(classes = [AppConfiguration::class, KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class])
@MockBean(ObjectMapper::class)
class AppConfigurationTest {
@Nested
@ -65,10 +65,7 @@ class AppConfigurationTest {
"app.kafka.group-id=test"
]
)
@MockBeans(value = [
MockBean(ObjectMapper::class),
MockBean(RequestRepository::class)
])
@MockBean(RequestRepository::class)
inner class AppConfigurationKafkaTest(private val context: ApplicationContext) {
@Test
@ -99,4 +96,24 @@ class AppConfigurationTest {
}
@Nested
@TestPropertySource(
properties = [
"app.transformations[0].path=consent.status",
"app.transformations[0].from=rejected",
"app.transformations[0].to=accept",
]
)
inner class AppConfigurationTransformationTest(private val context: ApplicationContext) {
@Test
fun shouldRecognizeTransformations() {
val appConfigProperties = context.getBean(AppConfigProperties::class.java)
assertThat(appConfigProperties).isNotNull
assertThat(appConfigProperties.transformations).hasSize(1)
}
}
}

11
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt
View File

@ -24,7 +24,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties
@ConfigurationProperties(AppConfigProperties.NAME)
data class AppConfigProperties(
var bwhcUri: String?,
var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN
var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN,
var transformations: List<TransformationProperties> = listOf()
) {
companion object {
const val NAME = "app"
@ -78,4 +79,10 @@ data class KafkaTargetProperties(
enum class PseudonymGenerator {
BUILDIN,
GPAS
}
}
data class TransformationProperties(
val path: String,
val from: String,
val to: String
)

16
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
View File

@ -25,6 +25,9 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.services.Transformation
import dev.dnpm.etl.processor.services.TransformationService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
@ -41,6 +44,8 @@ import reactor.core.publisher.Sinks
)
class AppConfiguration {
private val logger = LoggerFactory.getLogger(AppConfiguration::class.java)
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator {
@ -71,5 +76,16 @@ class AppConfiguration {
return Sinks.many().multicast().directBestEffort()
}
@Bean
fun transformationService(
objectMapper: ObjectMapper,
configProperties: AppConfigProperties
): TransformationService {
logger.info("Apply ${configProperties.transformations.size} transformation rules")
return TransformationService(objectMapper, configProperties.transformations.map {
Transformation.of(it.path) from it.from to it.to
})
}
}

3
src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt
View File

@ -38,6 +38,7 @@ import java.util.*
@Service
class RequestProcessor(
private val pseudonymizeService: PseudonymizeService,
private val transformationService: TransformationService,
private val sender: MtbFileSender,
private val requestService: RequestService,
private val objectMapper: ObjectMapper,
@ -50,7 +51,7 @@ class RequestProcessor(
mtbFile pseudonymizeWith pseudonymizeService
val request = MtbFileSender.MtbFileRequest(requestId, mtbFile)
val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
requestService.save(
Request(

81
src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt Normal file
View File

@ -0,0 +1,81 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.PathNotFoundException
import de.ukw.ccc.bwhc.dto.MtbFile
class TransformationService(private val objectMapper: ObjectMapper, private val transformations: List<Transformation>) {
fun transform(mtbFile: MtbFile): MtbFile {
var json = objectMapper.writeValueAsString(mtbFile)
transformations.forEach { transformation ->
val jsonPath = JsonPath.parse(json)
try {
val before = transformation.path.substringBeforeLast(".")
val last = transformation.path.substringAfterLast(".")
val existingValue = if (transformation.existingValue is Number) transformation.existingValue else transformation.existingValue.toString()
val newValue = if (transformation.newValue is Number) transformation.newValue else transformation.newValue.toString()
jsonPath.set("$.$before.[?]$last", newValue, {
it.item(HashMap::class.java)[last] == existingValue
})
} catch (e: PathNotFoundException) {
// Ignore
}
json = jsonPath.jsonString()
}
return objectMapper.readValue(json, MtbFile::class.java)
}
}
class Transformation private constructor(internal val path: String) {
lateinit var existingValue: Any
private set
lateinit var newValue: Any
private set
infix fun from(value: Any): Transformation {
this.existingValue = value
return this
}
infix fun to(value: Any): Transformation {
this.newValue = value
return this
}
companion object {
fun of(path: String): Transformation {
return Transformation(path)
}
}
}

21
src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt
View File

@ -37,6 +37,7 @@ import org.mockito.Mockito.*
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.argumentCaptor
import org.mockito.kotlin.whenever
import org.springframework.context.ApplicationEventPublisher
import java.time.Instant
import java.util.*
@ -46,6 +47,7 @@ import java.util.*
class RequestProcessorTest {
private lateinit var pseudonymizeService: PseudonymizeService
private lateinit var transformationService: TransformationService
private lateinit var sender: MtbFileSender
private lateinit var requestService: RequestService
private lateinit var applicationEventPublisher: ApplicationEventPublisher
@ -55,11 +57,13 @@ class RequestProcessorTest {
@BeforeEach
fun setup(
@Mock pseudonymizeService: PseudonymizeService,
@Mock transformationService: TransformationService,
@Mock sender: RestMtbFileSender,
@Mock requestService: RequestService,
@Mock applicationEventPublisher: ApplicationEventPublisher
) {
this.pseudonymizeService = pseudonymizeService
this.transformationService = transformationService
this.sender = sender
this.requestService = requestService
this.applicationEventPublisher = applicationEventPublisher
@ -68,6 +72,7 @@ class RequestProcessorTest {
requestProcessor = RequestProcessor(
pseudonymizeService,
transformationService,
sender,
requestService,
objectMapper,
@ -98,6 +103,10 @@ class RequestProcessorTest {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
@ -153,6 +162,10 @@ class RequestProcessorTest {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
@ -212,6 +225,10 @@ class RequestProcessorTest {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
@ -271,6 +288,10 @@ class RequestProcessorTest {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()

95
src/test/kotlin/dev/dnpm/etl/processor/services/TransformationServiceTest.kt Normal file
View File

@ -0,0 +1,95 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.Diagnosis
import de.ukw.ccc.bwhc.dto.Icd10
import de.ukw.ccc.bwhc.dto.MtbFile
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
class TransformationServiceTest {
private lateinit var service: TransformationService
@BeforeEach
fun setup() {
this.service = TransformationService(
ObjectMapper(), listOf(
Transformation.of("consent.status") from Consent.Status.ACTIVE to Consent.Status.REJECTED,
Transformation.of("diagnoses[*].icd10.version") from "2013" to "2014",
)
)
}
@Test
fun shouldTransformMtbFile() {
val mtbFile = MtbFile.builder().withDiagnoses(
listOf(
Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also {
it.version = "2013"
}).build()
)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual).isNotNull
assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014")
}
@Test
fun shouldOnlyTransformGivenValues() {
val mtbFile = MtbFile.builder().withDiagnoses(
listOf(
Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also {
it.version = "2013"
}).build(),
Diagnosis.builder().withId("5678").withIcd10(Icd10("F79.8").also {
it.version = "2019"
}).build()
)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual).isNotNull
assertThat(actual.diagnoses[0].icd10.code).isEqualTo("F79.9")
assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014")
assertThat(actual.diagnoses[1].icd10.code).isEqualTo("F79.8")
assertThat(actual.diagnoses[1].icd10.version).isEqualTo("2019")
}
@Test
fun shouldTransformMtbFileWithConsentEnum() {
val mtbFile = MtbFile.builder().withConsent(
Consent("123", "456", Consent.Status.ACTIVE)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual.consent).isNotNull
assertThat(actual.consent.status).isEqualTo(Consent.Status.REJECTED)
}
}