1
0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00

Compare commits

...

9 Commits

35 changed files with 3511 additions and 772 deletions

View File

@ -52,8 +52,6 @@ Soll noch das alte bwHC-Backend verwendet werden, so ist die Umgebungsvariable
In Versionen des ETL-Processors **nach Version 0.10** werden die folgenden Konfigurationsoptionen entfernt:
* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Nutzen Sie hier, wie unter [_Integration eines eigenen Root CA
Zertifikats_](#integration-eines-eigenen-root-ca-zertifikats) beschrieben, das Einbinden eigener Zertifikate.
* `APP_KAFKA_TOPIC`: Nutzen Sie nun die Konfigurationsoption `APP_KAFKA_OUTPUT_TOPIC`
* `APP_KAFKA_RESPONSE_TOPIC`: Nutzen Sie nun die Konfigurationsoption `APP_KAFKA_OUTPUT_RESPONSE_TOPIC`
@ -68,13 +66,11 @@ Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgen
* `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Präfix - `UNKNOWN`, wenn nicht gesetzt
* `APP_PSEUDONYMIZE_GENERATOR`: `BUILDIN` oder `GPAS` - `BUILDIN`, wenn nicht gesetzt
**Hinweise**:
**Hinweis**
* Der alte Konfigurationsparameter `APP_PSEUDONYMIZER` mit den Werten `GPAS` oder `BUILDIN` sollte nicht mehr verwendet
werden.
* Die Pseudonymisierung erfolgt im ETL-Prozessor nur für die Patienten-ID.
Andere IDs werden mithilfe des standortbezogenen Präfixes (erneut) anonymisiert, um für den aktuellen Kontext nicht
vergleichbare IDs bereitzustellen.
Die Pseudonymisierung erfolgt im ETL-Prozessor nur für die Patienten-ID.
Andere IDs werden mithilfe des standortbezogenen Präfixes (erneut) anonymisiert, um für den aktuellen Kontext nicht
vergleichbare IDs bereitzustellen.
#### Eingebaute Anonymisierung
@ -90,13 +86,6 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri
* `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname
* `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername
* `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort
* ~~`APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`~~: **Veraltet** - Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
**Wird in nach Version 0.10 entfernt**
Der Konfigurationsparameter `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION` sollte nicht mehr verwendet werden und wird nach
Version 0.10 entfernt.
Stattdessen sollte das Root Zertifikat wie unter [_Integration eines eigenen Root CA
Zertifikats_](#integration-eines-eigenen-root-ca-zertifikats) beschrieben eingebunden werden.
### Anmeldung mit einem Passwort
@ -226,9 +215,7 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an DNP
Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic übermittelt wird:
* `APP_KAFKA_OUTPUT_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen.
Ersetzt ~~`APP_KAFKA_TOPIC`~~, **welches nach Version 0.10 entfernt wird**.
* `APP_KAFKA_OUTPUT_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response".
Ersetzt ~~`APP_KAFKA_RESPONSE_TOPIC`~~, **welches nach Version 0.10 entfernt wird**.
* `APP_KAFKA_GROUP_ID`: Kafka GroupID des Consumers. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_group".
* `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste

View File

@ -5,7 +5,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
war
id("org.springframework.boot") version "3.3.10"
id("org.springframework.boot") version "3.4.4"
id("io.spring.dependency-management") version "1.1.7"
kotlin("jvm") version "1.9.25"
kotlin("plugin.spring") version "1.9.25"
@ -13,12 +13,12 @@ plugins {
}
group = "dev.dnpm"
version = "0.10.0-SNAPSHOT"
version = "0.11.0-SNAPSHOT"
var versions = mapOf(
"bwhc-dto-java" to "0.4.0",
"mtb-dto" to "0.1.0-SNAPSHOT",
"hapi-fhir" to "7.6.0",
"commons-compress" to "1.26.2",
"mockito-kotlin" to "5.4.0",
"archunit" to "1.3.0",
// Webjars
@ -49,9 +49,18 @@ configurations {
compileOnly {
extendsFrom(configurations.annotationProcessor.get())
}
all {
resolutionStrategy {
cacheChangingModulesFor(5, "minutes")
}
}
}
repositories {
maven {
url = uri("https://git.dnpm.dev/api/packages/public-snapshots/maven")
}
maven {
url = uri("https://git.dnpm.dev/api/packages/public/maven")
}
@ -73,6 +82,7 @@ dependencies {
implementation("commons-codec:commons-codec")
implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")
implementation("de.ukw.ccc:bwhc-dto-java:${versions["bwhc-dto-java"]}")
implementation("dev.pcvolkmer.mv64e:mtb-dto:${versions["mtb-dto"]}") { isChanging = true }
implementation("ca.uhn.hapi.fhir:hapi-fhir-base:${versions["hapi-fhir"]}")
implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}")
implementation("org.apache.httpcomponents.client5:httpclient5")
@ -99,10 +109,8 @@ dependencies {
integrationTestImplementation("org.testcontainers:junit-jupiter")
integrationTestImplementation("org.testcontainers:postgresql")
integrationTestImplementation("com.tngtech.archunit:archunit:${versions["archunit"]}")
integrationTestImplementation("net.sourceforge.htmlunit:htmlunit")
integrationTestImplementation("org.htmlunit:htmlunit")
integrationTestImplementation("org.springframework:spring-webflux")
// Override dependency version from org.testcontainers:junit-jupiter - CVE-2024-26308, CVE-2024-25710
integrationTestImplementation("org.apache.commons:commons-compress:${versions["commons-compress"]}")
}
tasks.withType<KotlinCompile> {
@ -119,8 +127,9 @@ tasks.withType<Test> {
}
}
task<Test>("integrationTest") {
tasks.register<Test>("integrationTest") {
description = "Runs integration tests"
group = "verification"
testClassesDirs = sourceSets["integrationTest"].output.classesDirs
classpath = sourceSets["integrationTest"].runtimeClasspath

View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.output.BwhcV1MtbFileRequest
import dev.dnpm.etl.processor.output.MtbFileSender
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
@ -33,10 +34,10 @@ import org.mockito.kotlin.*
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.context.ApplicationContext
import org.springframework.http.MediaType
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.post
@ -45,7 +46,7 @@ import org.testcontainers.junit.jupiter.Testcontainers
@Testcontainers
@ExtendWith(SpringExtension::class)
@SpringBootTest
@MockBean(MtbFileSender::class)
@MockitoBean(types = [MtbFileSender::class])
@TestPropertySource(
properties = [
"app.rest.uri=http://example.com",
@ -73,7 +74,7 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
)
inner class TransformationTest {
@MockBean
@MockitoBean
private lateinit var mtbFileSender: MtbFileSender
@Autowired
@ -91,7 +92,7 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
fun mtbFileIsTransformed() {
doAnswer {
MtbFileSender.Response(RequestStatus.SUCCESS)
}.whenever(mtbFileSender).send(any<MtbFileSender.MtbFileRequest>())
}.whenever(mtbFileSender).send(any<BwhcV1MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
@ -134,9 +135,9 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
}
}
val captor = argumentCaptor<MtbFileSender.MtbFileRequest>()
val captor = argumentCaptor<BwhcV1MtbFileRequest>()
verify(mtbFileSender).send(captor.capture())
assertThat(captor.firstValue.mtbFile.diagnoses).hasSize(1).allMatch { diagnosis ->
assertThat(captor.firstValue.content.diagnoses).hasSize(1).allMatch { diagnosis ->
diagnosis.icd10.version == "2014"
}
}

View File

@ -26,9 +26,9 @@ import dev.dnpm.etl.processor.output.KafkaMtbFileSender
import dev.dnpm.etl.processor.output.RestMtbFileSender
import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.security.TokenRepository
import dev.dnpm.etl.processor.security.TokenService
import dev.dnpm.etl.processor.services.RequestProcessor
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
@ -36,24 +36,25 @@ import org.junit.jupiter.api.assertThrows
import org.springframework.beans.factory.NoSuchBeanDefinitionException
import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.boot.test.mock.mockito.MockBeans
import org.springframework.context.ApplicationContext
import org.springframework.retry.support.RetryTemplate
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
@SpringBootTest
@ContextConfiguration(classes = [
@ContextConfiguration(
classes = [
AppConfiguration::class,
AppSecurityConfiguration::class,
KafkaAutoConfiguration::class,
AppKafkaConfiguration::class,
AppRestConfiguration::class
])
@MockBean(ObjectMapper::class)
]
)
@MockitoBean(types = [ObjectMapper::class])
@TestPropertySource(
properties = [
"app.pseudonymize.generator=BUILDIN",
@ -86,7 +87,7 @@ class AppConfigurationTest {
"app.kafka.group-id=test"
]
)
@MockBean(RequestRepository::class)
@MockitoBean(types = [RequestRepository::class])
inner class AppConfigurationKafkaTest(private val context: ApplicationContext) {
@Test
@ -145,7 +146,7 @@ class AppConfigurationTest {
"app.kafka.group-id=test"
]
)
@MockBean(RequestProcessor::class)
@MockitoBean(types = [RequestProcessor::class])
inner class AppConfigurationUsingKafkaInputTest(private val context: ApplicationContext) {
@Test
@ -181,40 +182,7 @@ class AppConfigurationTest {
@Nested
@TestPropertySource(
properties = [
"app.pseudonymize.generator=",
"app.pseudonymizer=buildin",
]
)
inner class AppConfigurationPseudonymizerBuildinTest(private val context: ApplicationContext) {
@Test
fun shouldUseConfiguredGenerator() {
assertThat(context.getBean(AnonymizingGenerator::class.java)).isNotNull
}
}
@Nested
@TestPropertySource(
properties = [
"app.pseudonymize.generator=",
"app.pseudonymizer=gpas",
]
)
inner class AppConfigurationPseudonymizerGpasTest(private val context: ApplicationContext) {
@Test
fun shouldUseConfiguredGenerator() {
assertThat(context.getBean(GpasPseudonymGenerator::class.java)).isNotNull
}
}
@Nested
@TestPropertySource(
properties = [
"app.pseudonymize.generator=buildin",
"app.pseudonymizer=",
"app.pseudonymize.generator=buildin"
]
)
inner class AppConfigurationPseudonymizeGeneratorBuildinTest(private val context: ApplicationContext) {
@ -229,8 +197,7 @@ class AppConfigurationTest {
@Nested
@TestPropertySource(
properties = [
"app.pseudonymize.generator=gpas",
"app.pseudonymizer=",
"app.pseudonymize.generator=gpas"
]
)
inner class AppConfigurationPseudonymizeGeneratorGpasTest(private val context: ApplicationContext) {
@ -248,11 +215,13 @@ class AppConfigurationTest {
"app.security.enable-tokens=true"
]
)
@MockBeans(value = [
MockBean(InMemoryUserDetailsManager::class),
MockBean(PasswordEncoder::class),
MockBean(TokenRepository::class)
])
@MockitoBean(
types = [
InMemoryUserDetailsManager::class,
PasswordEncoder::class,
TokenRepository::class
]
)
inner class AppConfigurationTokenEnabledTest(private val context: ApplicationContext) {
@Test
@ -263,11 +232,13 @@ class AppConfigurationTest {
}
@Nested
@MockBeans(value = [
MockBean(InMemoryUserDetailsManager::class),
MockBean(PasswordEncoder::class),
MockBean(TokenRepository::class)
])
@MockitoBean(
types = [
InMemoryUserDetailsManager::class,
PasswordEncoder::class,
TokenRepository::class
]
)
inner class AppConfigurationTokenDisabledTest(private val context: ApplicationContext) {
@Test

View File

@ -37,13 +37,13 @@ import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.http.MediaType
import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository
import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous
import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.delete
@ -57,7 +57,7 @@ import org.springframework.test.web.servlet.post
AppSecurityConfiguration::class
]
)
@MockBean(TokenRepository::class, RequestProcessor::class)
@MockitoBean(types = [TokenRepository::class, RequestProcessor::class])
@TestPropertySource(
properties = [
"app.pseudonymize.generator=BUILDIN",
@ -91,7 +91,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -104,7 +104,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -117,7 +117,7 @@ class MtbFileRestControllerTest {
status { isUnauthorized() }
}
verify(requestProcessor, never()).processMtbFile(any())
verify(requestProcessor, never()).processMtbFile(any<MtbFile>())
}
@Test
@ -130,7 +130,7 @@ class MtbFileRestControllerTest {
status { isForbidden() }
}
verify(requestProcessor, never()).processMtbFile(any())
verify(requestProcessor, never()).processMtbFile(any<MtbFile>())
}
@Test
@ -156,7 +156,7 @@ class MtbFileRestControllerTest {
}
@Nested
@MockBean(UserRoleRepository::class, ClientRegistrationRepository::class)
@MockitoBean(types = [UserRoleRepository::class, ClientRegistrationRepository::class])
@TestPropertySource(
properties = [
"app.pseudonymize.generator=BUILDIN",
@ -177,7 +177,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -190,7 +190,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
}

View File

@ -27,8 +27,8 @@ import org.junit.jupiter.api.extension.ExtendWith
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.data.jdbc.DataJdbcTest
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.transaction.annotation.Transactional
import org.testcontainers.junit.jupiter.Testcontainers
@ -39,7 +39,7 @@ import java.time.Instant
@DataJdbcTest
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Transactional
@MockBean(MtbFileSender::class)
@MockitoBean(types = [MtbFileSender::class])
@TestPropertySource(
properties = [
"app.pseudonymize.generator=buildin",

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -47,10 +47,9 @@ class GpasPseudonymGeneratorTest {
fun setup() {
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
val gPasConfigProperties = GPasConfigProperties(
"http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate",
"https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate",
"test",
null,
null,
null
)
@ -63,7 +62,7 @@ class GpasPseudonymGeneratorTest {
fun shouldReturnExpectedPseudonym() {
this.mockRestServiceServer.expect {
method(HttpMethod.POST)
requestTo("http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
requestTo("https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
}.andRespond {
withStatus(HttpStatus.OK).body(getDummyResponseBody("1234", "test", "test1234ABCDEF567890"))
.createResponse(it)
@ -76,7 +75,7 @@ class GpasPseudonymGeneratorTest {
fun shouldThrowExceptionIfGpasNotAvailable() {
this.mockRestServiceServer.expect {
method(HttpMethod.POST)
requestTo("http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
requestTo("https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
}.andRespond {
withException(IOException("Simulated IO error")).createResponse(it)
}
@ -88,7 +87,7 @@ class GpasPseudonymGeneratorTest {
fun shouldThrowExceptionIfGpasDoesNotReturn2xxResponse() {
this.mockRestServiceServer.expect {
method(HttpMethod.POST)
requestTo("http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
requestTo("https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
}.andRespond {
withStatus(HttpStatus.FOUND)
.header(HttpHeaders.LOCATION, "https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")

View File

@ -31,8 +31,8 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.transaction.annotation.Transactional
import org.testcontainers.junit.jupiter.Testcontainers
@ -42,7 +42,7 @@ import java.time.Instant
@ExtendWith(SpringExtension::class)
@SpringBootTest
@Transactional
@MockBean(MtbFileSender::class)
@MockitoBean(types = [MtbFileSender::class])
@TestPropertySource(
properties = [
"app.pseudonymize.generator=buildin",

View File

@ -19,8 +19,6 @@
package dev.dnpm.etl.processor.web
import com.gargoylesoftware.htmlunit.WebClient
import com.gargoylesoftware.htmlunit.html.HtmlPage
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
@ -29,11 +27,13 @@ import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.security.Role
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.security.TokenService
import dev.dnpm.etl.processor.services.TransformationService
import dev.dnpm.etl.processor.security.UserRoleService
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.TransformationService
import org.assertj.core.api.Assertions.assertThat
import org.htmlunit.WebClient
import org.htmlunit.html.HtmlPage
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
@ -46,7 +46,6 @@ import org.mockito.kotlin.verify
import org.mockito.kotlin.whenever
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.http.HttpHeaders
import org.springframework.http.MediaType
import org.springframework.http.MediaType.TEXT_EVENT_STREAM
@ -55,6 +54,7 @@ import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequ
import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.reactive.server.WebTestClient
import org.springframework.test.web.servlet.*
@ -81,14 +81,16 @@ abstract class MockSink : Sinks.Many<Boolean>
"app.pseudonymize.generator=BUILDIN"
]
)
@MockBean(name = "configsUpdateProducer", classes = [MockSink::class])
@MockBean(
@MockitoBean(name = "configsUpdateProducer", types = [MockSink::class])
@MockitoBean(
types = [
Generator::class,
MtbFileSender::class,
RequestProcessor::class,
TransformationService::class,
GPasConnectionCheckService::class,
RestConnectionCheckService::class,
RestConnectionCheckService::class
]
)
class ConfigControllerTest {
@ -143,8 +145,10 @@ class ConfigControllerTest {
"app.security.admin-user=admin"
]
)
@MockBean(
@MockitoBean(
types = [
TokenService::class
]
)
inner class WithTokensEnabled {
private lateinit var tokenService: TokenService
@ -252,8 +256,10 @@ class ConfigControllerTest {
"app.security.admin-password={noop}very-secret"
]
)
@MockBean(
@MockitoBean(
types = [
UserRoleService::class
]
)
inner class WithUserRolesEnabled {
private lateinit var userRoleService: UserRoleService

View File

@ -19,8 +19,6 @@
package dev.dnpm.etl.processor.web
import com.gargoylesoftware.htmlunit.WebClient
import com.gargoylesoftware.htmlunit.html.HtmlPage
import dev.dnpm.etl.processor.*
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
@ -30,6 +28,8 @@ import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.services.RequestService
import org.assertj.core.api.Assertions.assertThat
import org.htmlunit.WebClient
import org.htmlunit.html.HtmlPage
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
@ -40,13 +40,13 @@ import org.mockito.kotlin.any
import org.mockito.kotlin.whenever
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.data.domain.Page
import org.springframework.data.domain.PageImpl
import org.springframework.data.domain.Pageable
import org.springframework.security.test.context.support.WithMockUser
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.get
@ -71,8 +71,8 @@ import java.util.*
"app.security.admin-password={noop}very-secret"
]
)
@MockBean(
RequestService::class
@MockitoBean(
types = [RequestService::class]
)
class HomeControllerTest {

View File

@ -19,21 +19,21 @@
package dev.dnpm.etl.processor.web
import com.gargoylesoftware.htmlunit.WebClient
import com.gargoylesoftware.htmlunit.html.HtmlPage
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.security.TokenService
import org.assertj.core.api.Assertions.assertThat
import org.htmlunit.WebClient
import org.htmlunit.html.HtmlPage
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.junit.jupiter.MockitoExtension
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.get
@ -56,8 +56,8 @@ import org.springframework.test.web.servlet.htmlunit.MockMvcWebClientBuilder
"app.security.enable-tokens=true"
]
)
@MockBean(
TokenService::class,
@MockitoBean(
types = [TokenService::class]
)
class LoginControllerTest {

View File

@ -19,9 +19,9 @@
package dev.dnpm.etl.processor.web
import com.gargoylesoftware.htmlunit.WebClient
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import org.htmlunit.WebClient
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith

View File

@ -41,10 +41,10 @@ import org.mockito.kotlin.doAnswer
import org.mockito.kotlin.whenever
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.http.MediaType.TEXT_EVENT_STREAM
import org.springframework.test.context.ContextConfiguration
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.reactive.server.WebTestClient
import org.springframework.test.web.servlet.MockMvc
@ -74,8 +74,8 @@ import java.time.temporal.ChronoUnit
"app.security.admin-password={noop}very-secret"
]
)
@MockBean(
RequestService::class
@MockitoBean(
types = [RequestService::class]
)
class StatisticsRestControllerTest {

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -21,16 +21,10 @@ package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.security.Role
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.boot.context.properties.DeprecatedConfigurationProperty
@ConfigurationProperties(AppConfigProperties.NAME)
data class AppConfigProperties(
var bwhcUri: String?,
@get:DeprecatedConfigurationProperty(
reason = "Deprecated in favor of 'app.pseudonymize.generator'",
replacement = "app.pseudonymize.generator"
)
var pseudonymizer: PseudonymGenerator = PseudonymGenerator.BUILDIN,
var transformations: List<TransformationProperties> = listOf(),
var maxRetryAttempts: Int = 3,
var duplicationDetection: Boolean = true
@ -56,10 +50,6 @@ data class GPasConfigProperties(
val target: String = "etl-processor",
val username: String?,
val password: String?,
@get:DeprecatedConfigurationProperty(
reason = "Deprecated in favor of including Root CA"
)
val sslCaLocation: String?
) {
companion object {
const val NAME = "app.pseudonymize.gpas"
@ -82,18 +72,8 @@ data class RestTargetProperties(
data class KafkaProperties(
val inputTopic: String?,
val outputTopic: String = "etl-processor",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputTopic"
)
val topic: String = outputTopic,
val outputResponseTopic: String = "${outputTopic}_response",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputResponseTopic"
)
val responseTopic: String = outputResponseTopic,
val groupId: String = "${topic}_group",
val groupId: String = "${outputTopic}_group",
val servers: String = ""
) {
companion object {

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -32,12 +32,6 @@ import dev.dnpm.etl.processor.security.TokenRepository
import dev.dnpm.etl.processor.security.TokenService
import dev.dnpm.etl.processor.services.Transformation
import dev.dnpm.etl.processor.services.TransformationService
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
import org.apache.hc.client5.http.socket.ConnectionSocketFactory
import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory
import org.apache.hc.core5.http.config.RegistryBuilder
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@ -45,7 +39,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.data.jdbc.repository.config.AbstractJdbcConfiguration
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory
import org.springframework.retry.RetryCallback
import org.springframework.retry.RetryContext
import org.springframework.retry.RetryListener
@ -58,13 +51,6 @@ import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.web.client.HttpClientErrorException
import org.springframework.web.client.RestTemplate
import reactor.core.publisher.Sinks
import java.io.BufferedInputStream
import java.io.FileInputStream
import java.security.KeyStore
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import javax.net.ssl.SSLContext
import javax.net.ssl.TrustManagerFactory
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -90,18 +76,6 @@ class AppConfiguration {
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator {
try {
if (!configProperties.sslCaLocation.isNullOrBlank()) {
return GpasPseudonymGenerator(
configProperties,
retryTemplate,
createCustomGpasRestTemplate(configProperties)
)
}
} catch (e: Exception) {
throw RuntimeException(e)
}
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate)
}
@ -111,92 +85,6 @@ class AppConfiguration {
return AnonymizingGenerator()
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator {
try {
if (!configProperties.sslCaLocation.isNullOrBlank()) {
return GpasPseudonymGenerator(
configProperties,
retryTemplate,
createCustomGpasRestTemplate(configProperties)
)
}
} catch (e: Exception) {
throw RuntimeException(e)
}
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate)
}
private fun createCustomGpasRestTemplate(configProperties: GPasConfigProperties): RestTemplate {
fun getSslContext(certificateLocation: String): SSLContext? {
val ks = KeyStore.getInstance(KeyStore.getDefaultType())
val fis = FileInputStream(certificateLocation)
val ca = CertificateFactory.getInstance("X.509")
.generateCertificate(BufferedInputStream(fis)) as X509Certificate
ks.load(null, null)
ks.setCertificateEntry(1.toString(), ca)
val tmf = TrustManagerFactory.getInstance(
TrustManagerFactory.getDefaultAlgorithm()
)
tmf.init(ks)
val sslContext = SSLContext.getInstance("TLS")
sslContext.init(null, tmf.trustManagers, null)
return sslContext
}
fun getCustomRestTemplate(customSslContext: SSLContext): RestTemplate {
val sslsf = SSLConnectionSocketFactory(customSslContext)
val socketFactoryRegistry = RegistryBuilder.create<ConnectionSocketFactory>()
.register("https", sslsf).register("http", PlainConnectionSocketFactory()).build()
val connectionManager = BasicHttpClientConnectionManager(
socketFactoryRegistry
)
val httpClient = HttpClients.custom()
.setConnectionManager(connectionManager).build()
val requestFactory = HttpComponentsClientHttpRequestFactory(
httpClient
)
return RestTemplate(requestFactory)
}
try {
if (!configProperties.sslCaLocation.isNullOrBlank()) {
val customSslContext = getSslContext(configProperties.sslCaLocation)
logger.warn(
String.format(
"%s has been initialized with SSL certificate %s. This is deprecated in favor of including Root CA.",
this.javaClass.name, configProperties.sslCaLocation
)
)
if (customSslContext != null) {
return getCustomRestTemplate(customSslContext)
}
}
} catch (e: Exception) {
throw RuntimeException(e)
}
throw RuntimeException("Custom SSL configuration for gPAS not usable")
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN")
@ConditionalOnMissingBean
@Bean
fun buildinPseudonymGeneratorOnDeprecatedProperty(): Generator {
return AnonymizingGenerator()
}
@Bean
fun pseudonymizeService(
generator: Generator,

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -71,7 +71,7 @@ class AppKafkaConfiguration {
kafkaProperties: KafkaProperties,
kafkaResponseProcessor: KafkaResponseProcessor
): KafkaMessageListenerContainer<String, String> {
val containerProperties = ContainerProperties(kafkaProperties.responseTopic)
val containerProperties = ContainerProperties(kafkaProperties.outputResponseTopic)
containerProperties.messageListener = kafkaResponseProcessor
return KafkaMessageListenerContainer(consumerFactory, containerProperties)
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -87,9 +87,14 @@ class AppSecurityConfiguration(
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun filterChainOidc(http: HttpSecurity, passwordEncoder: PasswordEncoder, userRoleRepository: UserRoleRepository, sessionRegistry: SessionRegistry): SecurityFilterChain {
fun filterChainOidc(
http: HttpSecurity,
passwordEncoder: PasswordEncoder,
userRoleRepository: UserRoleRepository,
sessionRegistry: SessionRegistry
): SecurityFilterChain {
http {
authorizeRequests {
authorizeHttpRequests {
authorize("/configs/**", hasRole("ADMIN"))
authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN", "USER"))
authorize("/report/**", hasAnyRole("ADMIN", "USER"))
@ -127,13 +132,22 @@ class AppSecurityConfiguration(
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun grantedAuthoritiesMapper(userRoleRepository: UserRoleRepository, appSecurityConfigProperties: SecurityConfigProperties): GrantedAuthoritiesMapper {
fun grantedAuthoritiesMapper(
userRoleRepository: UserRoleRepository,
appSecurityConfigProperties: SecurityConfigProperties
): GrantedAuthoritiesMapper {
return GrantedAuthoritiesMapper { grantedAuthority ->
grantedAuthority.filterIsInstance<OidcUserAuthority>()
.onEach {
val userRole = userRoleRepository.findByUsername(it.userInfo.preferredUsername)
if (userRole.isEmpty) {
userRoleRepository.save(UserRole(null, it.userInfo.preferredUsername, appSecurityConfigProperties.defaultNewUserRole))
userRoleRepository.save(
UserRole(
null,
it.userInfo.preferredUsername,
appSecurityConfigProperties.defaultNewUserRole
)
)
}
}
.map {
@ -147,7 +161,7 @@ class AppSecurityConfiguration(
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "false", matchIfMissing = true)
fun filterChain(http: HttpSecurity, passwordEncoder: PasswordEncoder): SecurityFilterChain {
http {
authorizeRequests {
authorizeHttpRequests {
authorize("/configs/**", hasRole("ADMIN"))
authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN"))
authorize("/report/**", hasRole("ADMIN"))

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -22,11 +22,13 @@ package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.slf4j.LoggerFactory
import org.springframework.http.MediaType
import org.springframework.kafka.listener.MessageListener
class KafkaInputListener(
@ -35,10 +37,29 @@ class KafkaInputListener(
) : MessageListener<String, String> {
private val logger = LoggerFactory.getLogger(KafkaInputListener::class.java)
override fun onMessage(data: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java)
override fun onMessage(record: ConsumerRecord<String, String>) {
when (guessMimeType(record)) {
MediaType.APPLICATION_JSON_VALUE -> handleBwhcMessage(record)
CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE -> handleDnpmV2Message(record)
else -> {
/* ignore other messages */
}
}
}
private fun guessMimeType(record: ConsumerRecord<String, String>): String {
if (record.headers().headers("contentType").toList().isEmpty()) {
// Fallback if no contentType set (old behavior)
return MediaType.APPLICATION_JSON_VALUE
}
return record.headers().headers("contentType")?.firstOrNull()?.value().contentToString()
}
private fun handleBwhcMessage(record: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(record.value(), MtbFile::class.java)
val patientId = PatientId(mtbFile.patient.id)
val firstRequestIdHeader = data.headers().headers("requestId")?.firstOrNull()
val firstRequestIdHeader = record.headers().headers("requestId")?.firstOrNull()
val requestId = if (null != firstRequestIdHeader) {
RequestId(String(firstRequestIdHeader.value()))
} else {
@ -61,4 +82,10 @@ class KafkaInputListener(
}
}
}
private fun handleDnpmV2Message(record: ConsumerRecord<String, String>) {
// Do not handle DNPM-V2 for now
logger.warn("Ignoring MTB File in DNPM V2 format: Not implemented yet")
}
}

View File

@ -21,9 +21,12 @@ package dev.dnpm.etl.processor.input
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.slf4j.LoggerFactory
import org.springframework.http.MediaType
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.*
@ -40,19 +43,26 @@ class MtbFileRestController(
return ResponseEntity.ok("Test")
}
@PostMapping
@PostMapping( consumes = [ MediaType.APPLICATION_JSON_VALUE ] )
fun mtbFile(@RequestBody mtbFile: MtbFile): ResponseEntity<Unit> {
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
logger.debug("Accepted MTB File for processing")
logger.debug("Accepted MTB File (bwHC V1) for processing")
requestProcessor.processMtbFile(mtbFile)
} else {
logger.debug("Accepted MTB File and process deletion")
logger.debug("Accepted MTB File (bwHC V1) and process deletion")
val patientId = PatientId(mtbFile.patient.id)
requestProcessor.processDeletion(patientId)
}
return ResponseEntity.accepted().build()
}
@PostMapping( consumes = [ CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE] )
fun mtbFile(@RequestBody mtbFile: Mtb): ResponseEntity<Unit> {
logger.debug("Accepted MTB File (DNPM V2) for processing")
requestProcessor.processMtbFile(mtbFile)
return ResponseEntity.accepted().build()
}
@DeleteMapping(path = ["{patientId}"])
fun deleteData(@PathVariable patientId: String): ResponseEntity<Unit> {
logger.debug("Accepted patient ID to process deletion")

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -22,10 +22,12 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.apache.kafka.clients.producer.ProducerRecord
import org.slf4j.LoggerFactory
import org.springframework.http.MediaType
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.retry.support.RetryTemplate
@ -38,14 +40,20 @@ class KafkaMtbFileSender(
private val logger = LoggerFactory.getLogger(KafkaMtbFileSender::class.java)
override fun send(request: MtbFileSender.MtbFileRequest): MtbFileSender.Response {
override fun <T> send(request: MtbFileRequest<T>): MtbFileSender.Response {
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaProperties.topic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, request.mtbFile))
)
val record =
ProducerRecord(kafkaProperties.outputTopic, key(request), objectMapper.writeValueAsString(request))
when (request) {
is BwhcV1MtbFileRequest -> record.headers()
.add("contentType", MediaType.APPLICATION_JSON_VALUE.toByteArray())
is DnpmV2MtbFileRequest -> record.headers()
.add("contentType", CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE.toByteArray())
}
val result = kafkaTemplate.send(record)
if (result.get() != null) {
logger.debug("Sent file via KafkaMtbFileSender")
MtbFileSender.Response(RequestStatus.UNKNOWN)
@ -59,7 +67,7 @@ class KafkaMtbFileSender(
}
}
override fun send(request: MtbFileSender.DeleteRequest): MtbFileSender.Response {
override fun send(request: DeleteRequest): MtbFileSender.Response {
val dummyMtbFile = MtbFile.builder()
.withConsent(
Consent.builder()
@ -71,12 +79,15 @@ class KafkaMtbFileSender(
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaProperties.topic,
val record =
ProducerRecord(
kafkaProperties.outputTopic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, dummyMtbFile))
// Always use old BwhcV1FileRequest with Consent REJECT
objectMapper.writeValueAsString(BwhcV1MtbFileRequest(request.requestId, dummyMtbFile))
)
val result = kafkaTemplate.send(record)
if (result.get() != null) {
logger.debug("Sent deletion request via KafkaMtbFileSender")
MtbFileSender.Response(RequestStatus.UNKNOWN)
@ -91,16 +102,15 @@ class KafkaMtbFileSender(
}
override fun endpoint(): String {
return "${this.kafkaProperties.servers} (${this.kafkaProperties.topic}/${this.kafkaProperties.responseTopic})"
return "${this.kafkaProperties.servers} (${this.kafkaProperties.outputTopic}/${this.kafkaProperties.outputResponseTopic})"
}
private fun key(request: MtbFileSender.MtbFileRequest): String {
return "{\"pid\": \"${request.mtbFile.patient.id}\"}"
private fun key(request: MtbRequest): String {
return when (request) {
is BwhcV1MtbFileRequest -> "{\"pid\": \"${request.content.patient.id}\"}"
is DnpmV2MtbFileRequest -> "{\"pid\": \"${request.content.patient.id}\"}"
is DeleteRequest -> "{\"pid\": \"${request.patientId.value}\"}"
else -> throw IllegalArgumentException("Unsupported request type: ${request::class.simpleName}")
}
private fun key(request: MtbFileSender.DeleteRequest): String {
return "{\"pid\": \"${request.patientId.value}\"}"
}
data class Data(val requestId: RequestId, val content: MtbFile)
}

View File

@ -19,25 +19,17 @@
package dev.dnpm.etl.processor.output
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.springframework.http.HttpStatusCode
interface MtbFileSender {
fun send(request: MtbFileRequest): Response
fun <T> send(request: MtbFileRequest<T>): Response
fun send(request: DeleteRequest): Response
fun endpoint(): String
data class Response(val status: RequestStatus, val body: String = "")
data class MtbFileRequest(val requestId: RequestId, val mtbFile: MtbFile)
data class DeleteRequest(val requestId: RequestId, val patientId: PatientPseudonym)
}
fun Int.asRequestStatus(): RequestStatus {

View File

@ -0,0 +1,59 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.output
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.pcvolkmer.mv64e.mtb.Mtb
interface MtbRequest {
val requestId: RequestId
}
sealed interface MtbFileRequest<out T> : MtbRequest {
override val requestId: RequestId
val content: T
fun patientPseudonym(): PatientPseudonym
}
data class BwhcV1MtbFileRequest(
override val requestId: RequestId,
override val content: MtbFile
) : MtbFileRequest<MtbFile> {
override fun patientPseudonym(): PatientPseudonym {
return PatientPseudonym(content.patient.id)
}
}
data class DnpmV2MtbFileRequest(
override val requestId: RequestId,
override val content: Mtb
) : MtbFileRequest<Mtb> {
override fun patientPseudonym(): PatientPseudonym {
return PatientPseudonym(content.patient.id)
}
}
data class DeleteRequest(
override val requestId: RequestId,
val patientId: PatientPseudonym
) : MtbRequest

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -19,10 +19,11 @@
package dev.dnpm.etl.processor.output
import dev.dnpm.etl.processor.config.RestTargetProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.config.RestTargetProperties
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.asRequestStatus
import org.slf4j.LoggerFactory
import org.springframework.http.HttpEntity
@ -46,11 +47,11 @@ abstract class RestMtbFileSender(
abstract fun deleteUrl(patientId: PatientPseudonym): String
override fun send(request: MtbFileSender.MtbFileRequest): MtbFileSender.Response {
override fun <T> send(request: MtbFileRequest<T>): MtbFileSender.Response {
try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val headers = getHttpHeaders()
val entityReq = HttpEntity(request.mtbFile, headers)
val headers = getHttpHeaders(request)
val entityReq = HttpEntity(request.content, headers)
val response = restTemplate.postForEntity(
sendUrl(),
entityReq,
@ -76,10 +77,10 @@ abstract class RestMtbFileSender(
return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung")
}
override fun send(request: MtbFileSender.DeleteRequest): MtbFileSender.Response {
override fun send(request: DeleteRequest): MtbFileSender.Response {
try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val headers = getHttpHeaders()
val headers = getHttpHeaders(request)
val entityReq = HttpEntity(null, headers)
restTemplate.delete(
deleteUrl(request.patientId),
@ -102,11 +103,15 @@ abstract class RestMtbFileSender(
return this.restTargetProperties.uri.orEmpty()
}
private fun getHttpHeaders(): HttpHeaders {
private fun getHttpHeaders(request: MtbRequest): HttpHeaders {
val username = restTargetProperties.username
val password = restTargetProperties.password
val headers = HttpHeaders()
headers.contentType = MediaType.APPLICATION_JSON
headers.contentType = when (request) {
is BwhcV1MtbFileRequest -> MediaType.APPLICATION_JSON
is DnpmV2MtbFileRequest -> CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON
else -> MediaType.APPLICATION_JSON
}
if (username.isNullOrBlank() || password.isNullOrBlank()) {
return headers

View File

@ -21,12 +21,12 @@ package dev.dnpm.etl.processor.pseudonym
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientId
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.apache.commons.codec.digest.DigestUtils
/** Replaces patient ID with generated patient pseudonym
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing patient pseudonymes
*/
infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
@ -49,7 +49,11 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
}
this.lastGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.molecularPathologyFindings?.forEach { it.patient = patientPseudonym }
this.molecularTherapies?.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
this.molecularTherapies?.forEach { molecularTherapy ->
molecularTherapy.history.forEach {
it.patient = patientPseudonym
}
}
this.ngsReports?.forEach { it.patient = patientPseudonym }
this.previousGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.rebiopsyRequests?.forEach { it.patient = patientPseudonym }
@ -63,7 +67,6 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
* Creates new hash of content IDs with given prefix except for patient IDs
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing rehashed content IDs
*/
infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService) {
@ -224,3 +227,89 @@ infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService)
}
}
}
/** Replaces patient ID with generated patient pseudonym
*
* @since 0.11.0
*
* @param pseudonymizeService The pseudonymizeService to be used
* @return The MTB file containing patient pseudonymes
*/
infix fun Mtb.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
val patientPseudonym = pseudonymizeService.patientPseudonym(PatientId(this.patient.id)).value
this.episodesOfCare?.forEach { it.patient.id = patientPseudonym }
this.carePlans?.forEach {
it.patient.id = patientPseudonym
it.rebiopsyRequests?.forEach { it.patient.id = patientPseudonym }
it.histologyReevaluationRequests?.forEach { it.patient.id = patientPseudonym }
it.medicationRecommendations.forEach { it.patient.id = patientPseudonym }
it.studyEnrollmentRecommendations?.forEach { it.patient.id = patientPseudonym }
it.procedureRecommendations?.forEach { it.patient.id = patientPseudonym }
it.geneticCounselingRecommendation.patient.id = patientPseudonym
}
this.diagnoses?.forEach { it.patient.id = patientPseudonym }
this.guidelineTherapies?.forEach { it.patient.id = patientPseudonym }
this.guidelineProcedures?.forEach { it.patient.id = patientPseudonym }
this.patient.id = patientPseudonym
this.claims?.forEach { it.patient.id = patientPseudonym }
this.claimResponses?.forEach { it.patient.id = patientPseudonym }
this.diagnoses?.forEach { it.patient.id = patientPseudonym }
this.histologyReports?.forEach {
it.patient.id = patientPseudonym
it.results.tumorMorphology?.patient?.id = patientPseudonym
it.results.tumorCellContent?.patient?.id = patientPseudonym
}
this.ngsReports?.forEach {
it.patient.id = patientPseudonym
it.results.simpleVariants?.forEach { it.patient.id = patientPseudonym }
it.results.copyNumberVariants?.forEach { it.patient.id = patientPseudonym }
it.results.dnaFusions?.forEach { it.patient.id = patientPseudonym }
it.results.rnaFusions?.forEach { it.patient.id = patientPseudonym }
it.results.tumorCellContent?.patient?.id = patientPseudonym
it.results.brcaness?.patient?.id = patientPseudonym
it.results.tmb?.patient?.id = patientPseudonym
it.results.hrdScore?.patient?.id = patientPseudonym
}
this.ihcReports?.forEach {
it.patient.id = patientPseudonym
it.results.msiMmr?.forEach { it.patient.id = patientPseudonym }
it.results.proteinExpression?.forEach { it.patient.id = patientPseudonym }
}
this.responses?.forEach { it.patient.id = patientPseudonym }
this.specimens?.forEach { it.patient.id = patientPseudonym }
this.priorDiagnosticReports?.forEach { it.patient.id = patientPseudonym }
this.performanceStatus.forEach { it.patient.id = patientPseudonym }
this.systemicTherapies.forEach {
it.history?.forEach {
it.patient.id = patientPseudonym
}
}
}
/**
* Creates new hash of content IDs with given prefix except for patient IDs
*
* @since 0.11.0
*
* @param pseudonymizeService The pseudonymizeService to be used
* @return The MTB file containing rehashed content IDs
*/
infix fun Mtb.anonymizeContentWith(pseudonymizeService: PseudonymizeService) {
val prefix = pseudonymizeService.prefix()
fun anonymize(id: String): String {
val hash = DigestUtils.sha256Hex("$prefix-$id").substring(0, 41).lowercase()
return "$prefix$hash"
}
this.episodesOfCare?.forEach {
it?.apply {
id = id?.let {
anonymize(it)
}
}
}
// TODO all other properties
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -27,10 +27,11 @@ import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.output.*
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.pseudonym.anonymizeContentWith
import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.apache.commons.codec.binary.Base32
import org.apache.commons.codec.digest.DigestUtils
import org.springframework.context.ApplicationEventPublisher
@ -55,29 +56,40 @@ class RequestProcessor(
fun processMtbFile(mtbFile: MtbFile, requestId: RequestId) {
val pid = PatientId(mtbFile.patient.id)
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = BwhcV1MtbFileRequest(requestId, transformationService.transform(mtbFile))
saveAndSend(request, pid)
}
val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
fun processMtbFile(mtbFile: Mtb) {
processMtbFile(mtbFile, randomRequestId())
}
val patientPseudonym = PatientPseudonym(request.mtbFile.patient.id)
fun processMtbFile(mtbFile: Mtb, requestId: RequestId) {
val pid = PatientId(mtbFile.patient.id)
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = DnpmV2MtbFileRequest(requestId, transformationService.transform(mtbFile))
saveAndSend(request, pid)
}
private fun <T> saveAndSend(request: MtbFileRequest<T>, pid: PatientId) {
requestService.save(
Request(
requestId,
patientPseudonym,
request.requestId,
request.patientPseudonym(),
pid,
fingerprint(request.mtbFile),
fingerprint(request),
RequestType.MTB_FILE,
RequestStatus.UNKNOWN
)
)
if (appConfigProperties.duplicationDetection && isDuplication(mtbFile)) {
if (appConfigProperties.duplicationDetection && isDuplication(request)) {
applicationEventPublisher.publishEvent(
ResponseEvent(
requestId,
request.requestId,
Instant.now(),
RequestStatus.DUPLICATION
)
@ -89,7 +101,7 @@ class RequestProcessor(
applicationEventPublisher.publishEvent(
ResponseEvent(
requestId,
request.requestId,
Instant.now(),
responseStatus.status,
when (responseStatus.status) {
@ -100,8 +112,11 @@ class RequestProcessor(
)
}
private fun isDuplication(pseudonymizedMtbFile: MtbFile): Boolean {
val patientPseudonym = PatientPseudonym(pseudonymizedMtbFile.patient.id)
private fun <T> isDuplication(pseudonymizedMtbFileRequest: MtbFileRequest<T>): Boolean {
val patientPseudonym = when (pseudonymizedMtbFileRequest) {
is BwhcV1MtbFileRequest -> PatientPseudonym(pseudonymizedMtbFileRequest.content.patient.id)
is DnpmV2MtbFileRequest -> PatientPseudonym(pseudonymizedMtbFileRequest.content.patient.id)
}
val lastMtbFileRequestForPatient =
requestService.lastMtbFileRequestForPatientPseudonym(patientPseudonym)
@ -109,7 +124,7 @@ class RequestProcessor(
return null != lastMtbFileRequestForPatient
&& !isLastRequestDeletion
&& lastMtbFileRequestForPatient.fingerprint == fingerprint(pseudonymizedMtbFile)
&& lastMtbFileRequestForPatient.fingerprint == fingerprint(pseudonymizedMtbFileRequest)
}
fun processDeletion(patientId: PatientId) {
@ -131,7 +146,7 @@ class RequestProcessor(
)
)
val responseStatus = sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym))
val responseStatus = sender.send(DeleteRequest(requestId, patientPseudonym))
applicationEventPublisher.publishEvent(
ResponseEvent(
@ -160,8 +175,11 @@ class RequestProcessor(
}
}
private fun fingerprint(mtbFile: MtbFile): Fingerprint {
return fingerprint(objectMapper.writeValueAsString(mtbFile))
private fun <T> fingerprint(request: MtbFileRequest<T>): Fingerprint {
return when (request) {
is BwhcV1MtbFileRequest -> fingerprint(objectMapper.writeValueAsString(request.content))
is DnpmV2MtbFileRequest -> fingerprint(objectMapper.writeValueAsString(request.content))
}
}
private fun fingerprint(s: String): Fingerprint {

View File

@ -23,10 +23,21 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.PathNotFoundException
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.pcvolkmer.mv64e.mtb.Mtb
class TransformationService(private val objectMapper: ObjectMapper, private val transformations: List<Transformation>) {
fun transform(mtbFile: MtbFile): MtbFile {
var json = objectMapper.writeValueAsString(mtbFile)
val json = transform(objectMapper.writeValueAsString(mtbFile))
return objectMapper.readValue(json, MtbFile::class.java)
}
fun transform(mtbFile: Mtb): Mtb {
val json = transform(objectMapper.writeValueAsString(mtbFile))
return objectMapper.readValue(json, Mtb::class.java)
}
private fun transform(content: String): String {
var json = content
transformations.forEach { transformation ->
val jsonPath = JsonPath.parse(json)
@ -48,7 +59,7 @@ class TransformationService(private val objectMapper: ObjectMapper, private val
json = jsonPath.jsonString()
}
return objectMapper.readValue(json, MtbFile::class.java)
return json
}
fun getTransformations(): List<Transformation> {

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -19,6 +19,7 @@
package dev.dnpm.etl.processor
import org.springframework.http.MediaType
import java.util.*
class Fingerprint(val value: String) {
@ -47,3 +48,16 @@ value class PatientId(val value: String)
value class PatientPseudonym(val value: String)
fun emptyPatientPseudonym() = PatientPseudonym("")
/**
* Custom MediaTypes
*
* @since 0.11.0
*/
object CustomMediaType {
val APPLICATION_VND_DNPM_V2_MTB_JSON = MediaType("application", "vnd.dnpm.v2.mtb+json")
const val APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE = "application/vnd.dnpm.v2.mtb+json"
val APPLICATION_VND_DNPM_V2_RD_JSON = MediaType("application", "vnd.dnpm.v2.rd+json")
const val APPLICATION_VND_DNPM_V2_RD_JSON_VALUE = "application/vnd.dnpm.v2.rd+json"
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeader
@ -63,9 +64,17 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
"",
this.objectMapper.writeValueAsString(mtbFile)
)
)
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -75,7 +84,15 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
"",
this.objectMapper.writeValueAsString(mtbFile)
)
)
verify(requestProcessor, times(1)).processDeletion(anyValueClass())
}
@ -89,10 +106,22 @@ class KafkaInputListenerTest {
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
ConsumerRecord(
"testtopic",
0,
0,
-1L,
TimestampType.NO_TIMESTAMP_TYPE,
-1,
-1,
"",
this.objectMapper.writeValueAsString(mtbFile),
headers,
Optional.empty()
)
)
verify(requestProcessor, times(1)).processMtbFile(any(), anyValueClass())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>(), anyValueClass())
}
@Test
@ -104,9 +133,52 @@ class KafkaInputListenerTest {
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
ConsumerRecord(
"testtopic",
0,
0,
-1L,
TimestampType.NO_TIMESTAMP_TYPE,
-1,
-1,
"",
this.objectMapper.writeValueAsString(mtbFile),
headers,
Optional.empty()
)
)
verify(requestProcessor, times(1)).processDeletion(anyValueClass(), anyValueClass())
}
@Test
fun shouldNotProcessDnpmV2Request() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
val headers = RecordHeaders(
listOf(
RecordHeader("requestId", UUID.randomUUID().toString().toByteArray()),
RecordHeader("contentType", CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE.toByteArray())
)
)
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
-1L,
TimestampType.NO_TIMESTAMP_TYPE,
-1,
-1,
"",
this.objectMapper.writeValueAsString(mtbFile),
headers,
Optional.empty()
)
)
verify(requestProcessor, times(0)).processDeletion(anyValueClass(), anyValueClass())
}
}

View File

@ -21,7 +21,9 @@ package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
@ -32,6 +34,7 @@ import org.mockito.Mockito.verify
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.anyValueClass
import org.springframework.core.io.ClassPathResource
import org.springframework.http.MediaType
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.delete
@ -70,7 +73,7 @@ class MtbFileRestControllerTest {
}
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -126,7 +129,7 @@ class MtbFileRestControllerTest {
}
}
verify(requestProcessor, times(1)).processMtbFile(any())
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
@ -155,6 +158,40 @@ class MtbFileRestControllerTest {
}
}
@Nested
inner class RequestsForDnpmDataModel21 {
private lateinit var mockMvc: MockMvc
private lateinit var requestProcessor: RequestProcessor
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
val controller = MtbFileRestController(requestProcessor)
this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build()
}
@Test
fun shouldRespondPostRequest() {
val mtbFileContent = ClassPathResource("mv64e-mtb-fake-patient.json").inputStream.readAllBytes().toString(Charsets.UTF_8)
mockMvc.post("/mtb") {
content = mtbFileContent
contentType = CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
}
companion object {
fun bwhcMtbFileContent(consentStatus: Consent.Status) = MtbFile.builder()
.withPatient(

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -21,20 +21,25 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.pcvolkmer.mv64e.mtb.*
import org.apache.kafka.clients.producer.ProducerRecord
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.mockito.ArgumentMatchers.anyString
import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.*
import org.springframework.http.MediaType
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.kafka.support.SendResult
import org.springframework.retry.policy.SimpleRetryPolicy
@ -45,6 +50,9 @@ import java.util.concurrent.ExecutionException
@ExtendWith(MockitoExtension::class)
class KafkaMtbFileSenderTest {
@Nested
inner class BwhcV1Record {
private lateinit var kafkaTemplate: KafkaTemplate<String, String>
private lateinit var kafkaMtbFileSender: KafkaMtbFileSender
@ -65,67 +73,69 @@ class KafkaMtbFileSenderTest {
}
@ParameterizedTest
@MethodSource("requestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldSendMtbFileRequestAndReturnExpectedState(testData: TestData) {
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
val response = kafkaMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile(Consent.Status.ACTIVE)))
val response = kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
assertThat(response.status).isEqualTo(testData.requestStatus)
}
@ParameterizedTest
@MethodSource("requestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldSendDeleteRequestAndReturnExpectedState(testData: TestData) {
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
val response = kafkaMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val response = kafkaMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(testData.requestStatus)
}
@Test
fun shouldSendMtbFileRequestWithCorrectKeyAndBody() {
fun shouldSendMtbFileRequestWithCorrectKeyAndHeaderAndBody() {
doAnswer {
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile(Consent.Status.ACTIVE)))
kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
val captor = argumentCaptor<String>()
verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture())
assertThat(captor.firstValue).isNotNull
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.secondValue).isNotNull
assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData(TEST_REQUEST_ID, Consent.Status.ACTIVE)))
val captor = argumentCaptor<ProducerRecord<String, String>>()
verify(kafkaTemplate, times(1)).send(captor.capture())
assertThat(captor.firstValue.key()).isNotNull
assertThat(captor.firstValue.key()).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.firstValue.headers().headers("contentType")).isNotNull
assertThat(captor.firstValue.headers().headers("contentType")?.firstOrNull()?.value()).isEqualTo(MediaType.APPLICATION_JSON_VALUE.toByteArray())
assertThat(captor.firstValue.value()).isNotNull
assertThat(captor.firstValue.value()).isEqualTo(objectMapper.writeValueAsString(bwhcV1kafkaRecordData(TEST_REQUEST_ID, Consent.Status.ACTIVE)))
}
@Test
fun shouldSendDeleteRequestWithCorrectKeyAndBody() {
doAnswer {
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
kafkaMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val captor = argumentCaptor<String>()
verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture())
assertThat(captor.firstValue).isNotNull
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.secondValue).isNotNull
assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData(TEST_REQUEST_ID, Consent.Status.REJECTED)))
val captor = argumentCaptor<ProducerRecord<String, String>>()
verify(kafkaTemplate, times(1)).send(captor.capture())
assertThat(captor.firstValue.key()).isNotNull
assertThat(captor.firstValue.key()).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.firstValue.value()).isNotNull
assertThat(captor.firstValue.value()).isEqualTo(objectMapper.writeValueAsString(bwhcV1kafkaRecordData(TEST_REQUEST_ID, Consent.Status.REJECTED)))
}
@ParameterizedTest
@MethodSource("requestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldRetryOnMtbFileKafkaSendError(testData: TestData) {
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
@ -136,9 +146,9 @@ class KafkaMtbFileSenderTest {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile(Consent.Status.ACTIVE)))
kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
val expectedCount = when (testData.exception) {
// OK - No Retry
@ -147,11 +157,11 @@ class KafkaMtbFileSenderTest {
else -> times(3)
}
verify(kafkaTemplate, expectedCount).send(anyString(), anyString(), anyString())
verify(kafkaTemplate, expectedCount).send(any<ProducerRecord<String, String>>())
}
@ParameterizedTest
@MethodSource("requestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldRetryOnDeleteKafkaSendError(testData: TestData) {
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
@ -162,9 +172,9 @@ class KafkaMtbFileSenderTest {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(anyString(), anyString(), anyString())
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
kafkaMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val expectedCount = when (testData.exception) {
// OK - No Retry
@ -173,14 +183,98 @@ class KafkaMtbFileSenderTest {
else -> times(3)
}
verify(kafkaTemplate, expectedCount).send(anyString(), anyString(), anyString())
verify(kafkaTemplate, expectedCount).send(any<ProducerRecord<String, String>>())
}
}
@Nested
inner class DnpmV2Record {
private lateinit var kafkaTemplate: KafkaTemplate<String, String>
private lateinit var kafkaMtbFileSender: KafkaMtbFileSender
private lateinit var objectMapper: ObjectMapper
@BeforeEach
fun setup(
@Mock kafkaTemplate: KafkaTemplate<String, String>
) {
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.objectMapper = ObjectMapper()
this.kafkaTemplate = kafkaTemplate
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldSendMtbFileRequestAndReturnExpectedState(testData: TestData) {
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
val response = kafkaMtbFileSender.send(DnpmV2MtbFileRequest(TEST_REQUEST_ID, dnpmV2MtbFile()))
assertThat(response.status).isEqualTo(testData.requestStatus)
}
@Test
fun shouldSendMtbFileRequestWithCorrectKeyAndHeaderAndBody() {
doAnswer {
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(DnpmV2MtbFileRequest(TEST_REQUEST_ID, dnpmV2MtbFile()))
val captor = argumentCaptor<ProducerRecord<String, String>>()
verify(kafkaTemplate, times(1)).send(captor.capture())
assertThat(captor.firstValue.key()).isNotNull
assertThat(captor.firstValue.key()).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.firstValue.headers().headers("contentType")).isNotNull
assertThat(captor.firstValue.headers().headers("contentType")?.firstOrNull()?.value()).isEqualTo(CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE.toByteArray())
assertThat(captor.firstValue.value()).isNotNull
assertThat(captor.firstValue.value()).isEqualTo(objectMapper.writeValueAsString(dnmpV2kafkaRecordData(TEST_REQUEST_ID)))
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldRetryOnMtbFileKafkaSendError(testData: TestData) {
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(DnpmV2MtbFileRequest(TEST_REQUEST_ID, dnpmV2MtbFile()))
val expectedCount = when (testData.exception) {
// OK - No Retry
null -> times(1)
// Request failed - Retry max 3 times
else -> times(3)
}
verify(kafkaTemplate, expectedCount).send(any<ProducerRecord<String, String>>())
}
}
companion object {
val TEST_REQUEST_ID = RequestId("TestId")
val TEST_PATIENT_PSEUDONYM = PatientPseudonym("PID")
fun mtbFile(consentStatus: Consent.Status): MtbFile {
fun bwhcV1MtbFile(consentStatus: Consent.Status): MtbFile {
return if (consentStatus == Consent.Status.ACTIVE) {
MtbFile.builder()
.withPatient(
@ -215,8 +309,31 @@ class KafkaMtbFileSenderTest {
}.build()
}
fun kafkaRecordData(requestId: RequestId, consentStatus: Consent.Status): KafkaMtbFileSender.Data {
return KafkaMtbFileSender.Data(requestId, mtbFile(consentStatus))
fun dnpmV2MtbFile(): Mtb = Mtb.builder()
.withPatient(
dev.pcvolkmer.mv64e.mtb.Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(CodingGender.builder().withCode(CodingGender.Code.MALE).build())
.build()
)
.withEpisodesOfCare(
listOf(
MTBEpisodeOfCare.builder()
.withId("1")
.withPatient(Reference("PID"))
.withPeriod(PeriodDate.builder().withStart("2023-08-08").build())
.build()
)
)
.build()
fun bwhcV1kafkaRecordData(requestId: RequestId, consentStatus: Consent.Status): MtbRequest {
return BwhcV1MtbFileRequest(requestId, bwhcV1MtbFile(consentStatus))
}
fun dnmpV2kafkaRecordData(requestId: RequestId): MtbRequest {
return DnpmV2MtbFileRequest(requestId, dnpmV2MtbFile())
}
data class TestData(val requestStatus: RequestStatus, val exception: Throwable? = null)

View File

@ -30,16 +30,16 @@ import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.Arguments
import org.junit.jupiter.params.provider.MethodSource
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.retry.policy.SimpleRetryPolicy
import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.test.web.client.ExpectedCount
import org.springframework.test.web.client.MockRestServiceServer
import org.springframework.test.web.client.match.MockRestRequestMatchers.method
import org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo
import org.springframework.test.web.client.match.MockRestRequestMatchers.*
import org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
import org.springframework.web.client.RestTemplate
@ -73,7 +73,7 @@ class RestBwhcMtbFileSenderTest {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ -84,11 +84,12 @@ class RestBwhcMtbFileSenderTest {
this.mockRestServiceServer
.expect(method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/mtbfile/MTBFile"))
.andExpect(header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile))
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ -118,7 +119,7 @@ class RestBwhcMtbFileSenderTest {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile))
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ -148,7 +149,7 @@ class RestBwhcMtbFileSenderTest {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}

View File

@ -22,6 +22,8 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.KotlinModule
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.config.AppConfigProperties
@ -29,25 +31,30 @@ import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.RestTargetProperties
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.output.RestBwhcMtbFileSenderTest.Companion
import dev.pcvolkmer.mv64e.mtb.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.retry.backoff.NoBackOffPolicy
import org.springframework.retry.policy.SimpleRetryPolicy
import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.test.web.client.ExpectedCount
import org.springframework.test.web.client.MockRestServiceServer
import org.springframework.test.web.client.match.MockRestRequestMatchers.method
import org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo
import org.springframework.test.web.client.match.MockRestRequestMatchers.*
import org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
import org.springframework.web.client.RestTemplate
class RestDipMtbFileSenderTest {
@Nested
inner class BwhcV1ContentRequest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var restMtbFileSender: RestMtbFileSender
@ -62,41 +69,28 @@ class RestDipMtbFileSenderTest {
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender = RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
this.restMtbFileSender =
RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
@ParameterizedTest
@MethodSource("deleteRequestWithResponseSource")
fun shouldReturnExpectedResponseForDelete(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.DELETE))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient/${TEST_PATIENT_PSEUDONYM.value}"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("mtbFileRequestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#mtbFileRequestWithResponseSource")
fun shouldReturnExpectedResponseForMtbFilePost(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient-record"))
.andExpect(header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile))
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("mtbFileRequestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#mtbFileRequestWithResponseSource")
fun shouldRetryOnMtbFileHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
@ -123,13 +117,89 @@ class RestDipMtbFileSenderTest {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest(TEST_REQUEST_ID, mtbFile))
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
}
@Nested
inner class DnpmV2ContentRequest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var restMtbFileSender: RestMtbFileSender
private var reportService = ReportService(ObjectMapper().registerModule(KotlinModule.Builder().build()))
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender = RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#mtbFileRequestWithResponseSource")
fun shouldReturnExpectedResponseForDnpmV2MtbFilePost(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient-record"))
.andExpect(header(HttpHeaders.CONTENT_TYPE, CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(DnpmV2MtbFileRequest(TEST_REQUEST_ID, dnpmV2MtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
}
@Nested
inner class DeleteRequest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var restMtbFileSender: RestMtbFileSender
private var reportService = ReportService(ObjectMapper().registerModule(KotlinModule.Builder().build()))
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#deleteRequestWithResponseSource")
fun shouldReturnExpectedResponseForDelete(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.DELETE))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient/${TEST_PATIENT_PSEUDONYM.value}"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("deleteRequestWithResponseSource")
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#deleteRequestWithResponseSource")
fun shouldRetryOnDeleteHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
@ -156,11 +226,13 @@ class RestDipMtbFileSenderTest {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(MtbFileSender.DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
}
companion object {
data class RequestWithResponse(
val httpStatus: HttpStatus,
@ -171,7 +243,7 @@ class RestDipMtbFileSenderTest {
val TEST_REQUEST_ID = RequestId("TestId")
val TEST_PATIENT_PSEUDONYM = PatientPseudonym("PID")
val mtbFile: MtbFile = MtbFile.builder()
val bwhcV1mtbFile: MtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("PID")
@ -195,6 +267,25 @@ class RestDipMtbFileSenderTest {
)
.build()
val dnpmV2MtbFile: Mtb = Mtb.builder()
.withPatient(
dev.pcvolkmer.mv64e.mtb.Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(CodingGender.builder().withCode(CodingGender.Code.MALE).build())
.build()
)
.withEpisodesOfCare(
listOf(
MTBEpisodeOfCare.builder()
.withId("1")
.withPatient(Reference("PID"))
.withPeriod(PeriodDate.builder().withStart("2023-08-08").build())
.build()
)
)
.build()
private const val ERROR_RESPONSE_BODY = "Sonstiger Fehler bei der Übertragung"
/**

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -21,7 +21,12 @@ package dev.dnpm.etl.processor.pseudonym
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.pcvolkmer.mv64e.mtb.MTBEpisodeOfCare
import dev.pcvolkmer.mv64e.mtb.Mtb
import dev.pcvolkmer.mv64e.mtb.PeriodDate
import dev.pcvolkmer.mv64e.mtb.Reference
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.extension.ExtendWith
@ -32,12 +37,15 @@ import org.mockito.kotlin.doAnswer
import org.mockito.kotlin.whenever
import org.springframework.core.io.ClassPathResource
const val FAKE_MTB_FILE_PATH = "fake_MTBFile.json"
const val CLEAN_PATIENT_ID = "5dad2f0b-49c6-47d8-a952-7b9e9e0f7549"
@ExtendWith(MockitoExtension::class)
class ExtensionsTest {
@Nested
inner class UsingBwhcDatamodel {
val FAKE_MTB_FILE_PATH = "fake_MTBFile.json"
val CLEAN_PATIENT_ID = "5dad2f0b-49c6-47d8-a952-7b9e9e0f7549"
private fun fakeMtbFile(): MtbFile {
val mtbFile = ClassPathResource(FAKE_MTB_FILE_PATH).inputStream
return ObjectMapper().readValue(mtbFile, MtbFile::class.java)
@ -191,8 +199,83 @@ class ExtensionsTest {
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id).isNotNull()
}
}
@Nested
inner class UsingDnpmV2Datamodel {
val FAKE_MTB_FILE_PATH = "mv64e-mtb-fake-patient.json"
val CLEAN_PATIENT_ID = "63f8fd7b-8127-4f3c-8843-aa9199e21c29"
private fun fakeMtbFile(): Mtb {
val mtbFile = ClassPathResource(FAKE_MTB_FILE_PATH).inputStream
return ObjectMapper().readValue(mtbFile, Mtb::class.java)
}
private fun Mtb.serialized(): String {
return ObjectMapper().writeValueAsString(this)
}
@Test
fun shouldNotContainCleanPatientId(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
val mtbFile = fakeMtbFile()
mtbFile.pseudonymizeWith(pseudonymizeService)
assertThat(mtbFile.patient.id).isEqualTo("PSEUDO-ID")
assertThat(mtbFile.serialized()).doesNotContain(CLEAN_PATIENT_ID)
}
@Test
fun shouldNotThrowExceptionOnNullValues(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = Mtb.builder()
.withPatient(
dev.pcvolkmer.mv64e.mtb.Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(null)
.build()
)
.withEpisodesOfCare(
listOf(
MTBEpisodeOfCare.builder()
.withId("1")
.withPatient(Reference("1"))
.withPeriod(PeriodDate.builder().withStart("2023-08-08").build())
.build()
)
)
.withClaims(null)
.withDiagnoses(null)
.withCarePlans(null)
.withClaimResponses(null)
.withHistologyReports(null)
.withNgsReports(null)
.withResponses(null)
.withSpecimens(null)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episodesOfCare).hasSize(1)
assertThat(mtbFile.episodesOfCare.map { it.id }).isNotNull
}
}
}

View File

@ -21,14 +21,19 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.*
import dev.dnpm.etl.processor.Fingerprint
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.BwhcV1MtbFileRequest
import dev.dnpm.etl.processor.output.DeleteRequest
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.output.RestMtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.randomRequestId
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
@ -109,7 +114,7 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
}.whenever(transformationService).transform(any<MtbFile>())
val mtbFile = MtbFile.builder()
.withPatient(
@ -168,7 +173,7 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
}.whenever(transformationService).transform(any<MtbFile>())
val mtbFile = MtbFile.builder()
.withPatient(
@ -223,7 +228,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<MtbFileSender.MtbFileRequest>())
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
doAnswer {
it.arguments[0] as String
@ -231,7 +236,7 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
}.whenever(transformationService).transform(any<MtbFile>())
val mtbFile = MtbFile.builder()
.withPatient(
@ -286,7 +291,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.ERROR)
}.whenever(sender).send(any<MtbFileSender.MtbFileRequest>())
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
doAnswer {
it.arguments[0] as String
@ -294,7 +299,7 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
}.whenever(transformationService).transform(any<MtbFile>())
val mtbFile = MtbFile.builder()
.withPatient(
@ -336,7 +341,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.UNKNOWN)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
}.whenever(sender).send(any<DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
@ -354,7 +359,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
}.whenever(sender).send(any<DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
@ -372,7 +377,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.ERROR)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
}.whenever(sender).send(any<DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
@ -404,11 +409,11 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
}.whenever(transformationService).transform(any<MtbFile>())
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<MtbFileSender.MtbFileRequest>())
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(

File diff suppressed because it is too large Load Diff