From cfdf41d550046b83193342406cd917770ab8d6ce Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 1 Mar 2024 07:27:58 +0100 Subject: [PATCH] feat: add config option to deactivate duplication check --- README.md | 5 ++ .../processor/config/AppConfigProperties.kt | 3 +- .../processor/services/RequestProcessor.kt | 6 ++- .../services/RequestProcessorTest.kt | 54 ++++++++++++++++++- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e1a0fe9..e033d69 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ Zudem ist eine minimalistische Weboberfläche integriert, die einen Einblick in ![Modell DNPM-ETL-Strecke](docs/etl.png) +### Duplikaterkennung + +Die Erkennung von Duplikaten ist normalerweise immer aktiv, kann jedoch über den Konfigurationsparameter +`APP_DUPLICATION_DETECTION=false` deaktiviert werden. + ### Datenübermittlung über HTTP/REST Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung direkt an das bwHC-Backend gesendet. diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 430648e..e8d6bfc 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -31,7 +31,8 @@ data class AppConfigProperties( ) var pseudonymizer: PseudonymGenerator = PseudonymGenerator.BUILDIN, var transformations: List = listOf(), - var maxRetryAttempts: Int = 3 + var maxRetryAttempts: Int = 3, + var duplicationDetection: Boolean = true ) { companion object { const val NAME = "app" diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index fd9a3f5..d0b6341 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.config.AppConfigProperties import dev.dnpm.etl.processor.monitoring.Report import dev.dnpm.etl.processor.monitoring.Request import dev.dnpm.etl.processor.monitoring.RequestStatus @@ -42,7 +43,8 @@ class RequestProcessor( private val sender: MtbFileSender, private val requestService: RequestService, private val objectMapper: ObjectMapper, - private val applicationEventPublisher: ApplicationEventPublisher + private val applicationEventPublisher: ApplicationEventPublisher, + private val appConfigProperties: AppConfigProperties ) { fun processMtbFile(mtbFile: MtbFile) { @@ -64,7 +66,7 @@ class RequestProcessor( ) ) - if (isDuplication(mtbFile)) { + if (appConfigProperties.duplicationDetection && isDuplication(mtbFile)) { applicationEventPublisher.publishEvent( ResponseEvent( requestId, diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 9aaa091..722ab7a 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.config.AppConfigProperties import dev.dnpm.etl.processor.monitoring.Request import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.monitoring.RequestType @@ -51,6 +52,7 @@ class RequestProcessorTest { private lateinit var sender: MtbFileSender private lateinit var requestService: RequestService private lateinit var applicationEventPublisher: ApplicationEventPublisher + private lateinit var appConfigProperties: AppConfigProperties private lateinit var requestProcessor: RequestProcessor @@ -67,6 +69,7 @@ class RequestProcessorTest { this.sender = sender this.requestService = requestService this.applicationEventPublisher = applicationEventPublisher + this.appConfigProperties = AppConfigProperties(null) val objectMapper = ObjectMapper() @@ -76,7 +79,8 @@ class RequestProcessorTest { sender, requestService, objectMapper, - applicationEventPublisher + applicationEventPublisher, + appConfigProperties ) } @@ -390,4 +394,52 @@ class RequestProcessorTest { assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) } + @Test + fun testShouldNotDetectMtbFileDuplicationIfDuplicationNotConfigured() { + this.appConfigProperties.duplicationDetection = false + + doAnswer { + it.arguments[0] as String + }.`when`(pseudonymizeService).patientPseudonym(any()) + + doAnswer { + it.arguments[0] + }.whenever(transformationService).transform(any()) + + doAnswer { + MtbFileSender.Response(status = RequestStatus.SUCCESS) + }.`when`(sender).send(any()) + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + } \ No newline at end of file