pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00
Code Issues Releases Activity

Merge pull request #47 from CCC-MF/issue_43

Browse Source 
feat: add config option to deactivate duplication check
This commit is contained in:
Paul-Christian Volkmer 2024-03-01 07:33:16 +01:00 committed by GitHub
commit 0b6decf88d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 64 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -15,6 +15,11 @@ Zudem ist eine minimalistische Weboberfläche integriert, die einen Einblick in
![Modell DNPM-ETL-Strecke](docs/etl.png)
### Duplikaterkennung
Die Erkennung von Duplikaten ist normalerweise immer aktiv, kann jedoch über den Konfigurationsparameter
`APP_DUPLICATION_DETECTION=false` deaktiviert werden.
### Datenübermittlung über HTTP/REST
Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung direkt an das bwHC-Backend gesendet.

3
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt
View File

@ -31,7 +31,8 @@ data class AppConfigProperties(
)
var pseudonymizer: PseudonymGenerator = PseudonymGenerator.BUILDIN,
var transformations: List<TransformationProperties> = listOf(),
var maxRetryAttempts: Int = 3
var maxRetryAttempts: Int = 3,
var duplicationDetection: Boolean = true
) {
companion object {
const val NAME = "app"

6
src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt
View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
@ -42,7 +43,8 @@ class RequestProcessor(
private val sender: MtbFileSender,
private val requestService: RequestService,
private val objectMapper: ObjectMapper,
private val applicationEventPublisher: ApplicationEventPublisher
private val applicationEventPublisher: ApplicationEventPublisher,
private val appConfigProperties: AppConfigProperties
) {
fun processMtbFile(mtbFile: MtbFile) {
@ -64,7 +66,7 @@ class RequestProcessor(
)
)
if (isDuplication(mtbFile)) {
if (appConfigProperties.duplicationDetection && isDuplication(mtbFile)) {
applicationEventPublisher.publishEvent(
ResponseEvent(
requestId,

54
src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt
View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
@ -51,6 +52,7 @@ class RequestProcessorTest {
private lateinit var sender: MtbFileSender
private lateinit var requestService: RequestService
private lateinit var applicationEventPublisher: ApplicationEventPublisher
private lateinit var appConfigProperties: AppConfigProperties
private lateinit var requestProcessor: RequestProcessor
@ -67,6 +69,7 @@ class RequestProcessorTest {
this.sender = sender
this.requestService = requestService
this.applicationEventPublisher = applicationEventPublisher
this.appConfigProperties = AppConfigProperties(null)
val objectMapper = ObjectMapper()
@ -76,7 +79,8 @@ class RequestProcessorTest {
sender,
requestService,
objectMapper,
applicationEventPublisher
applicationEventPublisher,
appConfigProperties
)
}
@ -390,4 +394,52 @@ class RequestProcessorTest {
assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR)
}
@Test
fun testShouldNotDetectMtbFileDuplicationIfDuplicationNotConfigured() {
this.appConfigProperties.duplicationDetection = false
doAnswer {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.`when`(sender).send(any<MtbFileSender.MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
this.requestProcessor.processMtbFile(mtbFile)
val eventCaptor = argumentCaptor<ResponseEvent>()
verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture())
assertThat(eventCaptor.firstValue).isNotNull
assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS)
}
}