From 51cf7a7917d7376d1e7c685b9c0e56d8929ad9e1 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 2 Aug 2023 11:52:11 +0200 Subject: [PATCH 01/50] Add processor to handle responses from Kafka topic --- README.md | 6 +- .../etl/processor/EtlProcessorApplication.kt | 1 - .../processor/config/AppConfigProperties.kt | 4 +- .../etl/processor/config/AppConfiguration.kt | 16 +--- .../processor/config/AppKafkaConfiguration.kt | 70 +++++++++++++++ .../dnpm/etl/processor/monitoring/Request.kt | 4 +- .../services/kafka/KafkaResponseProcessor.kt | 87 +++++++++++++++++++ src/main/resources/application-dev.yml | 9 +- 8 files changed, 176 insertions(+), 21 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt diff --git a/README.md b/README.md index db6ae44..1d5a7f3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,11 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an das Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic übermittelt wird: -* `APP_KAFKA_TOPIC`: Zu verwendendes Topic +* `APP_KAFKA_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen +* `APP_KAFKA_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response". +* `APP_KAFKA_GROUP_ID`: Kafka GroupID des Consumers. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_group". * `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste +Wird keine Rückantwort über Apache Kafka empfangen und gibt es keine weitere Möglichkeit den Status festzustellen, verbleibt der Status auf `UNKNOWN`. + Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/EtlProcessorApplication.kt b/src/main/kotlin/dev/dnpm/etl/processor/EtlProcessorApplication.kt index 0c4ab68..5d28c97 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/EtlProcessorApplication.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/EtlProcessorApplication.kt @@ -28,4 +28,3 @@ class EtlProcessorApplication fun main(args: Array) { runApplication(*args) } - diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 64be70d..6502a1b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -48,7 +48,7 @@ data class GPasConfigProperties( val password: String?, val sslCaLocation: String?, -) { + ) { companion object { const val NAME = "app.pseudonymize.gpas" } @@ -66,6 +66,8 @@ data class RestTargetProperties( @ConfigurationProperties(KafkaTargetProperties.NAME) data class KafkaTargetProperties( val topic: String = "etl-processor", + val responseTopic: String = "${topic}_response", + val groupId: String = "${topic}_group", val servers: String = "" ) { companion object { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index c677f2b..cbba1f1 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -21,7 +21,6 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper import dev.dnpm.etl.processor.monitoring.ReportService -import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.output.RestMtbFileSender import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator @@ -32,7 +31,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import org.springframework.kafka.core.KafkaTemplate import reactor.core.publisher.Sinks @Configuration @@ -60,7 +58,10 @@ class AppConfiguration { } @Bean - fun pseudonymizeService(generator: Generator, pseudonymizeConfigProperties: PseudonymizeConfigProperties): PseudonymizeService { + fun pseudonymizeService( + generator: Generator, + pseudonymizeConfigProperties: PseudonymizeConfigProperties + ): PseudonymizeService { return PseudonymizeService(generator, pseudonymizeConfigProperties) } @@ -70,15 +71,6 @@ class AppConfiguration { return RestMtbFileSender(restTargetProperties) } - @ConditionalOnProperty(value = ["app.kafka.topic", "app.kafka.servers"]) - @Bean - fun kafkaMtbFileSender( - kafkaTemplate: KafkaTemplate, - objectMapper: ObjectMapper - ): MtbFileSender { - return KafkaMtbFileSender(kafkaTemplate, objectMapper) - } - @Bean fun reportService(objectMapper: ObjectMapper): ReportService { return ReportService(objectMapper) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt new file mode 100644 index 0000000..f81d3fb --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt @@ -0,0 +1,70 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.config + +import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.output.KafkaMtbFileSender +import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.kafka.core.ConsumerFactory +import org.springframework.kafka.core.KafkaTemplate +import org.springframework.kafka.listener.ContainerProperties +import org.springframework.kafka.listener.KafkaMessageListenerContainer + +@Configuration +@EnableConfigurationProperties( + value = [KafkaTargetProperties::class] +) +@ConditionalOnProperty(value = ["app.kafka.topic", "app.kafka.servers"]) +class AppKafkaConfiguration { + + @Bean + fun kafkaMtbFileSender( + kafkaTemplate: KafkaTemplate, + objectMapper: ObjectMapper + ): MtbFileSender { + return KafkaMtbFileSender(kafkaTemplate, objectMapper) + } + + @Bean + fun kafkaListenerContainer( + consumerFactory: ConsumerFactory, + kafkaTargetProperties: KafkaTargetProperties, + kafkaResponseProcessor: KafkaResponseProcessor + ): KafkaMessageListenerContainer { + val containerProperties = ContainerProperties(kafkaTargetProperties.responseTopic); + containerProperties.messageListener = kafkaResponseProcessor + return KafkaMessageListenerContainer(consumerFactory, containerProperties) + } + + @Bean + fun kafkaResponseProcessor( + requestRepository: RequestRepository, + objectMapper: ObjectMapper + ): KafkaResponseProcessor { + return KafkaResponseProcessor(requestRepository, objectMapper) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt index ecd8219..c1d4d43 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt @@ -36,9 +36,9 @@ data class Request( val patientId: String, val pid: String, val fingerprint: String, - val status: RequestStatus, val type: RequestType, - val processedAt: Instant = Instant.now(), + var status: RequestStatus, + var processedAt: Instant = Instant.now(), @Embedded.Nullable var report: Report? = null ) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt new file mode 100644 index 0000000..f0c91cb --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -0,0 +1,87 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services.kafka + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.monitoring.Report +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.slf4j.LoggerFactory +import org.springframework.kafka.listener.MessageListener +import java.time.Instant + +class KafkaResponseProcessor( + private val requestRepository: RequestRepository, + private val objectMapper: ObjectMapper +) : MessageListener { + + private val logger = LoggerFactory.getLogger(KafkaResponseProcessor::class.java) + + override fun onMessage(data: ConsumerRecord) { + try { + val responseKey = objectMapper.readValue(data.key(), ResponseKey::class.java) + requestRepository.findByUuidEquals(responseKey.requestId).ifPresent { + val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) + when (responseBody.statusCode) { + 200 -> { + it.status = RequestStatus.SUCCESS + it.processedAt = Instant.ofEpochMilli(data.timestamp()) + requestRepository.save(it) + } + + 201 -> { + it.status = RequestStatus.WARNING + it.processedAt = Instant.ofEpochMilli(data.timestamp()) + it.report = Report( + "Warnungen über mangelhafte Daten", + responseBody.statusBody + ) + requestRepository.save(it) + } + + 400, 422 -> { + it.status = RequestStatus.ERROR + it.processedAt = Instant.ofEpochMilli(data.timestamp()) + it.report = Report( + "Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar", + responseBody.statusBody + ) + requestRepository.save(it) + } + + else -> { + logger.error("Cannot process Kafka response: Unknown response code!") + } + } + } + } catch (e: Exception) { + logger.error("Cannot process Kafka response", e) + } + } + + data class ResponseKey(val requestId: String) + + data class ResponseBody( + @JsonProperty("status code") val statusCode: Int, + @JsonProperty("status_body") val statusBody: String + ) +} \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 99e4bbf..551f3f8 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -5,10 +5,11 @@ spring: app: rest: - uri: http://localhost:9000/bwhc/etl/api/MTBFile - #kafka: - # topic: test - # servers: kafka:9092 + uri: http://localhost:9000/bwhc/etl/api + kafka: + topic: test + response-topic: test-response + servers: kafka:9092 server: port: 8000 From 35cb258b13543b37ce061f78eef4427e542ca72a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 2 Aug 2023 16:10:18 +0200 Subject: [PATCH 02/50] Do not return specific status code based on remote status code --- .../processor/services/RequestProcessor.kt | 17 ++++----------- .../etl/processor/web/MtbFileController.kt | 21 ++++++------------- src/main/resources/application.yml | 2 ++ 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 8588ebe..7d110b1 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -42,7 +42,7 @@ class RequestProcessor( private val logger = LoggerFactory.getLogger(RequestProcessor::class.java) - fun processMtbFile(mtbFile: MtbFile): RequestStatus { + fun processMtbFile(mtbFile: MtbFile) { val pid = mtbFile.patient.id val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) @@ -62,7 +62,7 @@ class RequestProcessor( ) ) statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) - return RequestStatus.DUPLICATION + return } val request = MtbFileSender.MtbFileRequest(UUID.randomUUID().toString(), pseudonymized) @@ -115,11 +115,9 @@ class RequestProcessor( ) statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) - - return requestStatus } - fun processDeletion(patientId: String): RequestStatus { + fun processDeletion(patientId: String) { val requestId = UUID.randomUUID().toString() try { @@ -178,10 +176,6 @@ class RequestProcessor( } ) ) - - statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) - - return overallRequestStatus } catch (e: Exception) { requestRepository.save( Request( @@ -194,11 +188,8 @@ class RequestProcessor( report = Report("Fehler bei der Pseudonymisierung") ) ) - - statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) - - return RequestStatus.ERROR } + statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) } private fun fingerprint(mtbFile: MtbFile): String { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt index a2cc953..cf0e693 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt @@ -20,7 +20,6 @@ package dev.dnpm.etl.processor.web import de.ukw.ccc.bwhc.dto.MtbFile -import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.services.RequestProcessor import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity @@ -35,24 +34,16 @@ class MtbFileController( @PostMapping(path = ["/mtbfile"]) fun mtbFile(@RequestBody mtbFile: MtbFile): ResponseEntity { - val requestStatus = requestProcessor.processMtbFile(mtbFile) - - return if (requestStatus == RequestStatus.ERROR) { - ResponseEntity.unprocessableEntity().build() - } else { - ResponseEntity.noContent().build() - } + logger.debug("Accepted MTB File for processing") + requestProcessor.processMtbFile(mtbFile) + return ResponseEntity.accepted().build() } @DeleteMapping(path = ["/mtbfile/{patientId}"]) fun deleteData(@PathVariable patientId: String): ResponseEntity { - val requestStatus = requestProcessor.processDeletion(patientId) - - return if (requestStatus == RequestStatus.ERROR) { - ResponseEntity.unprocessableEntity().build() - } else { - ResponseEntity.noContent().build() - } + logger.debug("Accepted patient ID to process deletion") + requestProcessor.processDeletion(patientId) + return ResponseEntity.accepted().build() } } \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 39acb37..72edde6 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,5 +3,7 @@ spring: bootstrap-servers: ${app.kafka.servers} template: default-topic: ${app.kafka.topic} + consumer: + group-id: ${app.kafka.group-id} flyway: locations: "classpath:db/migration/{vendor}" \ No newline at end of file From 70d4fa2f0ff4b38757cabc967b3f38a63674ed47 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 2 Aug 2023 16:10:53 +0200 Subject: [PATCH 03/50] Use duplication fingerprinting based on MTB file requests only --- .../kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 7d110b1..afac40b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -48,6 +48,7 @@ class RequestProcessor( val lastRequestForPatient = requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id) + .filter { it.type == RequestType.MTB_FILE } .firstOrNull { it.status == RequestStatus.SUCCESS || it.status == RequestStatus.WARNING } if (null != lastRequestForPatient && lastRequestForPatient.fingerprint == fingerprint(mtbFile)) { From 7f8b21efd2273bb7b4ee0d93ff4988bade2fa610 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 2 Aug 2023 16:23:16 +0200 Subject: [PATCH 04/50] Handle not parsable data quality reports --- .../dnpm/etl/processor/monitoring/ReportService.kt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index 6ee8ae9..8c31ede 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -20,6 +20,7 @@ package dev.dnpm.etl.processor.monitoring import com.fasterxml.jackson.annotation.JsonValue +import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.ObjectMapper @@ -33,9 +34,14 @@ class ReportService( } return try { objectMapper.readValue(dataQualityReport, DataQualityReport::class.java).issues - } catch (e: JsonMappingException) { - e.printStackTrace() - listOf() + } catch (e: Exception) { + val otherIssue = + Issue(Severity.ERROR, "Not parsable data quality report '$dataQualityReport'") + return when (e) { + is JsonMappingException -> listOf(otherIssue) + is JsonParseException -> listOf(otherIssue) + else -> throw e + } } } From 577509e6f2d502d4dbbf1a1b526c43497fde56b3 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 2 Aug 2023 16:53:09 +0200 Subject: [PATCH 05/50] Map 'status_code' and 'status code' to same data value --- .../etl/processor/services/kafka/KafkaResponseProcessor.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index f0c91cb..d6c4da6 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.services.kafka +import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper import dev.dnpm.etl.processor.monitoring.Report @@ -81,7 +82,7 @@ class KafkaResponseProcessor( data class ResponseKey(val requestId: String) data class ResponseBody( - @JsonProperty("status code") val statusCode: Int, + @JsonProperty("status_code") @JsonAlias("status code") val statusCode: Int, @JsonProperty("status_body") val statusBody: String ) } \ No newline at end of file From ac91620651daa2f9aa09709eaa0bb5a8f7222e71 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 3 Aug 2023 12:59:53 +0200 Subject: [PATCH 06/50] Use Map as status body since it contains JSON --- .../etl/processor/services/kafka/KafkaResponseProcessor.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index d6c4da6..547833c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -54,7 +54,7 @@ class KafkaResponseProcessor( it.processedAt = Instant.ofEpochMilli(data.timestamp()) it.report = Report( "Warnungen über mangelhafte Daten", - responseBody.statusBody + objectMapper.writeValueAsString(responseBody.statusBody) ) requestRepository.save(it) } @@ -64,7 +64,7 @@ class KafkaResponseProcessor( it.processedAt = Instant.ofEpochMilli(data.timestamp()) it.report = Report( "Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar", - responseBody.statusBody + objectMapper.writeValueAsString(responseBody.statusBody) ) requestRepository.save(it) } @@ -83,6 +83,6 @@ class KafkaResponseProcessor( data class ResponseBody( @JsonProperty("status_code") @JsonAlias("status code") val statusCode: Int, - @JsonProperty("status_body") val statusBody: String + @JsonProperty("status_body") val statusBody: Map ) } \ No newline at end of file From 3dcee41569b462a4b938380b5ac3b208728fb358 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 3 Aug 2023 15:14:49 +0200 Subject: [PATCH 07/50] Implement delete request using Apache Kafka This is implemented using a fake MTB file containing a rejected consent state and will be mapped to HTTP DELETE on kafka-to-bwhc consumer. --- .../processor/output/KafkaMtbFileSender.kt | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 55503cf..da25576 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -20,6 +20,8 @@ package dev.dnpm.etl.processor.output import com.fasterxml.jackson.databind.ObjectMapper +import de.ukw.ccc.bwhc.dto.Consent +import de.ukw.ccc.bwhc.dto.MtbFile import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate @@ -42,16 +44,38 @@ class KafkaMtbFileSender( } else { MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) } - } catch (e: Exception) { logger.error("An error occurred sending to kafka", e) MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } } - // TODO not yet implemented override fun send(request: MtbFileSender.DeleteRequest): MtbFileSender.Response { - return MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + val dummyMtbFile = MtbFile.builder() + .withConsent( + Consent.builder() + .withPatient(request.patientId) + .withStatus(Consent.Status.REJECTED) + .build() + ) + .build() + + return try { + val result = kafkaTemplate.sendDefault( + header(request), + objectMapper.writeValueAsString(dummyMtbFile) + ) + + if (result.get() != null) { + logger.debug("Sent deletion request via KafkaMtbFileSender") + MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) + } else { + MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) + } + } catch (e: Exception) { + logger.error("An error occurred sending to kafka", e) + MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + } } private fun header(request: MtbFileSender.MtbFileRequest): String { @@ -59,4 +83,9 @@ class KafkaMtbFileSender( "\"eid\": \"${request.mtbFile.episode.id}\", " + "\"requestId\": \"${request.requestId}\"}" } + + private fun header(request: MtbFileSender.DeleteRequest): String { + return "{\"pid\": \"${request.patientId}\", " + + "\"requestId\": \"${request.requestId}\"}" + } } \ No newline at end of file From ec76c775d9ab7c863cd0c55e6ba1232181f6775c Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 3 Aug 2023 16:04:31 +0200 Subject: [PATCH 08/50] Explicit producer topic configuration --- .../processor/config/AppKafkaConfiguration.kt | 5 +++-- .../processor/output/KafkaMtbFileSender.kt | 20 +++++++++++-------- .../services/kafka/KafkaResponseProcessor.kt | 13 ++++++++++++ src/main/resources/application.yml | 2 -- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt index f81d3fb..7adcb02 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt @@ -43,9 +43,10 @@ class AppKafkaConfiguration { @Bean fun kafkaMtbFileSender( kafkaTemplate: KafkaTemplate, + kafkaTargetProperties: KafkaTargetProperties, objectMapper: ObjectMapper ): MtbFileSender { - return KafkaMtbFileSender(kafkaTemplate, objectMapper) + return KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, objectMapper) } @Bean @@ -54,7 +55,7 @@ class AppKafkaConfiguration { kafkaTargetProperties: KafkaTargetProperties, kafkaResponseProcessor: KafkaResponseProcessor ): KafkaMessageListenerContainer { - val containerProperties = ContainerProperties(kafkaTargetProperties.responseTopic); + val containerProperties = ContainerProperties(kafkaTargetProperties.responseTopic) containerProperties.messageListener = kafkaResponseProcessor return KafkaMessageListenerContainer(consumerFactory, containerProperties) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index da25576..d903745 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -22,11 +22,13 @@ package dev.dnpm.etl.processor.output import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.config.KafkaTargetProperties import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate class KafkaMtbFileSender( private val kafkaTemplate: KafkaTemplate, + private val kafkaTargetProperties: KafkaTargetProperties, private val objectMapper: ObjectMapper ) : MtbFileSender { @@ -34,13 +36,14 @@ class KafkaMtbFileSender( override fun send(request: MtbFileSender.MtbFileRequest): MtbFileSender.Response { return try { - val result = kafkaTemplate.sendDefault( - header(request), + val result = kafkaTemplate.send( + kafkaTargetProperties.topic, + key(request), objectMapper.writeValueAsString(request.mtbFile) ) if (result.get() != null) { logger.debug("Sent file via KafkaMtbFileSender") - MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) + MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } else { MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) } @@ -61,14 +64,15 @@ class KafkaMtbFileSender( .build() return try { - val result = kafkaTemplate.sendDefault( - header(request), + val result = kafkaTemplate.send( + kafkaTargetProperties.topic, + key(request), objectMapper.writeValueAsString(dummyMtbFile) ) if (result.get() != null) { logger.debug("Sent deletion request via KafkaMtbFileSender") - MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) + MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } else { MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) } @@ -78,13 +82,13 @@ class KafkaMtbFileSender( } } - private fun header(request: MtbFileSender.MtbFileRequest): String { + private fun key(request: MtbFileSender.MtbFileRequest): String { return "{\"pid\": \"${request.mtbFile.patient.id}\", " + "\"eid\": \"${request.mtbFile.episode.id}\", " + "\"requestId\": \"${request.requestId}\"}" } - private fun header(request: MtbFileSender.DeleteRequest): String { + private fun key(request: MtbFileSender.DeleteRequest): String { return "{\"pid\": \"${request.patientId}\", " + "\"requestId\": \"${request.requestId}\"}" } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index 547833c..fd047d0 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -42,6 +42,9 @@ class KafkaResponseProcessor( val responseKey = objectMapper.readValue(data.key(), ResponseKey::class.java) requestRepository.findByUuidEquals(responseKey.requestId).ifPresent { val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) + + println("${responseBody.statusCode}") + when (responseBody.statusCode) { 200 -> { it.status = RequestStatus.SUCCESS @@ -69,6 +72,16 @@ class KafkaResponseProcessor( requestRepository.save(it) } + in 900..999 -> { + it.status = RequestStatus.ERROR + it.processedAt = Instant.ofEpochMilli(data.timestamp()) + it.report = Report( + "Fehler bei der Datenübertragung, keine Verbindung zum bwHC-Backend möglich", + objectMapper.writeValueAsString(responseBody.statusBody) + ) + requestRepository.save(it) + } + else -> { logger.error("Cannot process Kafka response: Unknown response code!") } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 72edde6..5cd47c0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,8 +1,6 @@ spring: kafka: bootstrap-servers: ${app.kafka.servers} - template: - default-topic: ${app.kafka.topic} consumer: group-id: ${app.kafka.group-id} flyway: From b14f2c1794fe41f9ec5e9e400d51c0fbf991953a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 3 Aug 2023 16:18:16 +0200 Subject: [PATCH 09/50] Add information about 'no connection' responses --- README.md | 7 +++++-- .../etl/processor/services/kafka/KafkaResponseProcessor.kt | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1d5a7f3..be6d6b5 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein * `APP_KAFKA_GROUP_ID`: Kafka GroupID des Consumers. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_group". * `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste -Wird keine Rückantwort über Apache Kafka empfangen und gibt es keine weitere Möglichkeit den Status festzustellen, verbleibt der Status auf `UNKNOWN`. +Wird keine Rückantwort über Apache Kafka empfangen und es gibt keine weitere Möglichkeit den Status festzustellen, verbleibt der Status auf `UNKNOWN`. -Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. \ No newline at end of file +Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. + +Lässt sich keine Verbindung zu dem bwHC-Backend aufbauen, wird eine Rückantwort mit Status-Code `900` erwartet, welchen es +für HTTP nicht gibt. \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index fd047d0..1e9263d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -43,8 +43,6 @@ class KafkaResponseProcessor( requestRepository.findByUuidEquals(responseKey.requestId).ifPresent { val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) - println("${responseBody.statusCode}") - when (responseBody.statusCode) { 200 -> { it.status = RequestStatus.SUCCESS From 459ad59c1d988a5b3ecc60d844f4fa6c9bce11f5 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 4 Aug 2023 11:43:23 +0200 Subject: [PATCH 10/50] Do not detect duplicates after deletion request --- .../dev/dnpm/etl/processor/services/RequestProcessor.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index afac40b..bdf2827 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -46,12 +46,15 @@ class RequestProcessor( val pid = mtbFile.patient.id val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) - val lastRequestForPatient = - requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id) + val allRequests = requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id) + + val lastMtbFileRequestForPatient = allRequests .filter { it.type == RequestType.MTB_FILE } .firstOrNull { it.status == RequestStatus.SUCCESS || it.status == RequestStatus.WARNING } - if (null != lastRequestForPatient && lastRequestForPatient.fingerprint == fingerprint(mtbFile)) { + val isLastRequestDeletion = allRequests.firstOrNull()?.type == RequestType.DELETE + + if (null != lastMtbFileRequestForPatient && lastMtbFileRequestForPatient.fingerprint == fingerprint(mtbFile) && !isLastRequestDeletion) { requestRepository.save( Request( patientId = pseudonymized.patient.id, From 3dea664999b803fb2f62c659fee977b28abc250b Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 12:46:04 +0200 Subject: [PATCH 11/50] Update Spring Boot dependencies --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index eecd959..f4f1cf7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,7 +2,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { war - id("org.springframework.boot") version "3.1.1" + id("org.springframework.boot") version "3.1.2" id("io.spring.dependency-management") version "1.1.0" kotlin("jvm") version "1.9.0" kotlin("plugin.spring") version "1.9.0" From 3039b4b2a7eb7963d0952a28ca5fd26328640223 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 13:23:18 +0200 Subject: [PATCH 12/50] Add basic Testcontainers test setup --- build.gradle.kts | 2 + .../processor/AbstractTestcontainerTest.kt | 45 +++++++++++++++++++ .../processor/BwhcMapperApplicationTests.kt | 37 +++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt diff --git a/build.gradle.kts b/build.gradle.kts index f4f1cf7..b59c028 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -50,6 +50,8 @@ dependencies { providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") + testImplementation("org.testcontainers:junit-jupiter") + testImplementation("org.testcontainers:postgresql") } tasks.withType { diff --git a/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt new file mode 100644 index 0000000..3bd934f --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt @@ -0,0 +1,45 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor + +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.junit.jupiter.Container + +abstract class AbstractTestcontainerTest { + + companion object { + @Container + val dbContainer = PostgreSQLContainer("postgres:10-alpine") + .withDatabaseName("test") + .withUsername("test") + .withPassword("test") ?: throw RuntimeException("Failed to create testcontainer!") + + @DynamicPropertySource + @JvmStatic + fun registerDynamicProperties(registry: DynamicPropertyRegistry) { + registry.add("spring.datasource.url", dbContainer::getJdbcUrl) + registry.add("spring.datasource.username", dbContainer::getUsername) + registry.add("spring.datasource.password", dbContainer::getPassword) + } + } + +} \ No newline at end of file diff --git a/src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt b/src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt new file mode 100644 index 0000000..efa6a66 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt @@ -0,0 +1,37 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.testcontainers.junit.jupiter.Testcontainers + +@Testcontainers +@ExtendWith(SpringExtension::class) +@SpringBootTest +class BwhcMapperApplicationTests : AbstractTestcontainerTest() { + + @Test + fun contextLoads() { + } + +} From 1fc09d691ea01415a21f1192cc7b1cf25bc0ac14 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 13:34:53 +0200 Subject: [PATCH 13/50] Rename test class to match applications main class name --- ...apperApplicationTests.kt => EtlProcessorApplicationTests.kt} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/kotlin/dev/dnpm/etl/processor/{BwhcMapperApplicationTests.kt => EtlProcessorApplicationTests.kt} (94%) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt b/src/test/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt similarity index 94% rename from src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt rename to src/test/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt index efa6a66..07a201b 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/BwhcMapperApplicationTests.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt @@ -28,7 +28,7 @@ import org.testcontainers.junit.jupiter.Testcontainers @Testcontainers @ExtendWith(SpringExtension::class) @SpringBootTest -class BwhcMapperApplicationTests : AbstractTestcontainerTest() { +class EtlProcessorApplicationTests : AbstractTestcontainerTest() { @Test fun contextLoads() { From bcc23f6b14436ba6f4585a583da6c236df68e25a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 14:50:12 +0200 Subject: [PATCH 14/50] Add RequestService to handle access to requests --- .../processor/services/RequestProcessor.kt | 17 +- .../etl/processor/services/RequestService.kt | 56 +++++ .../services/RequestServiceIntegrationTest.kt | 131 +++++++++++ .../processor/services/RequestServiceTest.kt | 205 ++++++++++++++++++ 4 files changed, 402 insertions(+), 7 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index bdf2827..e04e568 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -21,7 +21,10 @@ package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile -import dev.dnpm.etl.processor.monitoring.* +import dev.dnpm.etl.processor.monitoring.Report +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import org.apache.commons.codec.binary.Base32 @@ -35,7 +38,7 @@ import java.util.* class RequestProcessor( private val pseudonymizeService: PseudonymizeService, private val senders: List, - private val requestRepository: RequestRepository, + private val requestService: RequestService, private val objectMapper: ObjectMapper, private val statisticsUpdateProducer: Sinks.Many ) { @@ -46,7 +49,7 @@ class RequestProcessor( val pid = mtbFile.patient.id val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) - val allRequests = requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id) + val allRequests = requestService.allRequestsByPatientPseudonym(pseudonymized.patient.id) val lastMtbFileRequestForPatient = allRequests .filter { it.type == RequestType.MTB_FILE } @@ -55,7 +58,7 @@ class RequestProcessor( val isLastRequestDeletion = allRequests.firstOrNull()?.type == RequestType.DELETE if (null != lastMtbFileRequestForPatient && lastMtbFileRequestForPatient.fingerprint == fingerprint(mtbFile) && !isLastRequestDeletion) { - requestRepository.save( + requestService.save( Request( patientId = pseudonymized.patient.id, pid = pid, @@ -99,7 +102,7 @@ class RequestProcessor( RequestStatus.UNKNOWN } - requestRepository.save( + requestService.save( Request( uuid = request.requestId, patientId = request.mtbFile.patient.id, @@ -165,7 +168,7 @@ class RequestProcessor( RequestStatus.UNKNOWN } - requestRepository.save( + requestService.save( Request( uuid = requestId, patientId = patientPseudonym, @@ -181,7 +184,7 @@ class RequestProcessor( ) ) } catch (e: Exception) { - requestRepository.save( + requestService.save( Request( uuid = requestId, patientId = "???", diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt new file mode 100644 index 0000000..0f69910 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt @@ -0,0 +1,56 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType +import org.springframework.stereotype.Service + +@Service +class RequestService( + private val requestRepository: RequestRepository +) { + + fun save(request: Request) = requestRepository.save(request) + + fun allRequestsByPatientPseudonym(patientPseudonym: String) = requestRepository + .findAllByPatientIdOrderByProcessedAtDesc(patientPseudonym) + + fun lastMtbFileRequestForPatientPseudonym(patientPseudonym: String) = + Companion.lastMtbFileRequestForPatientPseudonym(allRequestsByPatientPseudonym(patientPseudonym)) + + fun isLastRequestDeletion(patientPseudonym: String) = + Companion.isLastRequestDeletion(allRequestsByPatientPseudonym(patientPseudonym)) + + companion object { + + fun lastMtbFileRequestForPatientPseudonym(allRequests: List) = allRequests + .filter { it.type == RequestType.MTB_FILE } + .sortedByDescending { it.processedAt } + .firstOrNull { it.status == RequestStatus.SUCCESS || it.status == RequestStatus.WARNING } + + fun isLastRequestDeletion(allRequests: List) = allRequests + .maxByOrNull { it.processedAt }?.type == RequestType.DELETE + + } + +} \ No newline at end of file diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt new file mode 100644 index 0000000..d71e011 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt @@ -0,0 +1,131 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import dev.dnpm.etl.processor.AbstractTestcontainerTest +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.springframework.transaction.annotation.Transactional +import org.testcontainers.junit.jupiter.Testcontainers +import java.time.Instant +import java.util.* + +@Testcontainers +@ExtendWith(SpringExtension::class) +@SpringBootTest +@Transactional +class RequestServiceIntegrationTest : AbstractTestcontainerTest() { + + private lateinit var requestRepository: RequestRepository + + private lateinit var requestService: RequestService + + @BeforeEach + fun setup( + @Autowired requestRepository: RequestRepository + ) { + this.requestRepository = requestRepository + this.requestService = RequestService(requestRepository) + } + + @Test + fun shouldResultInEmptyRequestList() { + val actual = requestService.allRequestsByPatientPseudonym("TEST_12345678901") + + assertThat(actual).isEmpty() + } + + private fun setupTestData() { + // Prepare DB + this.requestRepository.saveAll( + listOf( + Request( + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-07-07T02:00:00Z") + ), + // Should be ignored - wrong patient ID --> + Request( + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678902", + pid = "P2", + fingerprint = "0123456789abcdef2", + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-08-08T00:00:00Z") + ), + // <-- + Request( + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P2", + fingerprint = "0123456789abcdee1", + type = RequestType.DELETE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + ) + ) + } + + @Test + fun shouldResultInSortedRequestList() { + setupTestData() + + val actual = requestService.allRequestsByPatientPseudonym("TEST_12345678901") + + assertThat(actual).hasSize(2) + assertThat(actual[0].fingerprint).isEqualTo("0123456789abcdee1") + assertThat(actual[1].fingerprint).isEqualTo("0123456789abcdef1") + } + + @Test + fun shouldReturnDeleteRequestAsLastRequest() { + setupTestData() + + val actual = requestService.isLastRequestDeletion("TEST_12345678901") + + assertThat(actual).isTrue() + } + + @Test + fun shouldReturnLastMtbFileRequest() { + setupTestData() + + val actual = requestService.lastMtbFileRequestForPatientPseudonym("TEST_12345678901") + + assertThat(actual).isNotNull + assertThat(actual?.fingerprint).isEqualTo("0123456789abcdef1") + } + +} \ No newline at end of file diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt new file mode 100644 index 0000000..3e0a979 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt @@ -0,0 +1,205 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.Mockito.* +import org.mockito.junit.jupiter.MockitoExtension +import java.time.Instant +import java.util.* + +@ExtendWith(MockitoExtension::class) +class RequestServiceTest { + + private lateinit var requestRepository: RequestRepository + + private lateinit var requestService: RequestService + + private fun anyRequest() = any(Request::class.java) ?: Request( + id = 0L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_dummy", + pid = "PX", + fingerprint = "dummy", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + + @BeforeEach + fun setup( + @Mock requestRepository: RequestRepository + ) { + this.requestRepository = requestRepository + this.requestService = RequestService(requestRepository) + } + + @Test + fun shouldIndicateLastRequestIsDeleteRequest() { + val requests = listOf( + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.DELETE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ), + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678902", + pid = "P2", + fingerprint = "0123456789abcdef2", + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-08-08T00:00:00Z") + ) + ) + + val actual = RequestService.isLastRequestDeletion(requests) + + assertThat(actual).isTrue() + } + + @Test + fun shouldIndicateLastRequestIsNotDeleteRequest() { + val requests = listOf( + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.DELETE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-07-07T02:00:00Z") + ), + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678902", + pid = "P2", + fingerprint = "0123456789abcdef2", + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-08-08T00:00:00Z") + ) + ) + + val actual = RequestService.isLastRequestDeletion(requests) + + assertThat(actual).isFalse() + } + + @Test + fun shouldReturnPatientsLastRequest() { + val requests = listOf( + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.DELETE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-07-07T02:00:00Z") + ), + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678902", + pid = "P2", + fingerprint = "0123456789abcdef2", + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-08-08T00:00:00Z") + ) + ) + + val actual = RequestService.lastMtbFileRequestForPatientPseudonym(requests) + + assertThat(actual).isInstanceOf(Request::class.java) + assertThat(actual?.fingerprint).isEqualTo("0123456789abcdef2") + } + + @Test + fun shouldReturnNullIfNoRequests() { + val requests = listOf() + + val actual = RequestService.lastMtbFileRequestForPatientPseudonym(requests) + + assertThat(actual).isNull() + } + + @Test + fun saveShouldSaveRequestUsingRepository() { + doAnswer { + val obj = it.arguments[0] as Request + obj.copy(id = 1L) + }.`when`(requestRepository).save(anyRequest()) + + val request = Request( + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.DELETE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-07-07T02:00:00Z") + ) + + requestService.save(request) + + verify(requestRepository, times(1)).save(anyRequest()) + } + + @Test + fun allRequestsByPatientPseudonymShouldRequestAllRequestsForPatientPseudonym() { + requestService.allRequestsByPatientPseudonym("TEST_12345678901") + + verify(requestRepository, times(1)).findAllByPatientIdOrderByProcessedAtDesc(anyString()) + } + + @Test + fun lastMtbFileRequestForPatientPseudonymShouldRequestAllRequestsForPatientPseudonym() { + requestService.lastMtbFileRequestForPatientPseudonym("TEST_12345678901") + + verify(requestRepository, times(1)).findAllByPatientIdOrderByProcessedAtDesc(anyString()) + } + + @Test + fun isLastRequestDeletionShouldRequestAllRequestsForPatientPseudonym() { + requestService.isLastRequestDeletion("TEST_12345678901") + + verify(requestRepository, times(1)).findAllByPatientIdOrderByProcessedAtDesc(anyString()) + } + +} \ No newline at end of file From 4051b5094ca8daaa844803d2725b4094f3eed096 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 14:58:10 +0200 Subject: [PATCH 15/50] Keep database testcontainer alive until all tests are done --- .../dev/dnpm/etl/processor/AbstractTestcontainerTest.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt index 3bd934f..13b57d0 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt @@ -28,7 +28,7 @@ abstract class AbstractTestcontainerTest { companion object { @Container - val dbContainer = PostgreSQLContainer("postgres:10-alpine") + val dbContainer = CustomPostgreSQLContainer("postgres:10-alpine") .withDatabaseName("test") .withUsername("test") .withPassword("test") ?: throw RuntimeException("Failed to create testcontainer!") @@ -42,4 +42,10 @@ abstract class AbstractTestcontainerTest { } } +} + +class CustomPostgreSQLContainer(dockerImageName: String) : PostgreSQLContainer(dockerImageName) { + override fun stop() { + // Keep Testcontainer alive until JVM destroys it + } } \ No newline at end of file From b75328b74d361b7afd6197a5b240f5f76ce2c5e0 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 15:16:58 +0200 Subject: [PATCH 16/50] Move integration tests into own source-set --- build.gradle.kts | 24 +++++++++++++++++-- .../processor/AbstractTestcontainerTest.kt | 0 .../processor/EtlProcessorApplicationTests.kt | 0 .../services/RequestServiceIntegrationTest.kt | 0 4 files changed, 22 insertions(+), 2 deletions(-) rename src/{test => integrationTest}/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt (100%) rename src/{test => integrationTest}/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt (100%) rename src/{test => integrationTest}/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index b59c028..89df274 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,6 +15,18 @@ java { sourceCompatibility = JavaVersion.VERSION_17 } +sourceSets { + create("integrationTest") { + compileClasspath += sourceSets.main.get().output + runtimeClasspath += sourceSets.main.get().output + } +} + +val integrationTestImplementation: Configuration by configurations.getting { + extendsFrom(configurations.testImplementation.get()) + extendsFrom(configurations.runtimeOnly.get()) +} + configurations { compileOnly { extendsFrom(configurations.annotationProcessor.get()) @@ -50,8 +62,8 @@ dependencies { providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") - testImplementation("org.testcontainers:junit-jupiter") - testImplementation("org.testcontainers:postgresql") + integrationTestImplementation("org.testcontainers:junit-jupiter") + integrationTestImplementation("org.testcontainers:postgresql") } tasks.withType { @@ -65,3 +77,11 @@ tasks.withType { useJUnitPlatform() } +task("integrationTest") { + description = "Runs integration tests" + + testClassesDirs = sourceSets["integrationTest"].output.classesDirs + classpath = sourceSets["integrationTest"].runtimeClasspath + + shouldRunAfter("test") +} diff --git a/src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt similarity index 100% rename from src/test/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt rename to src/integrationTest/kotlin/dev/dnpm/etl/processor/AbstractTestcontainerTest.kt diff --git a/src/test/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt similarity index 100% rename from src/test/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt rename to src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt similarity index 100% rename from src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt rename to src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt From 422441a3b39806016a952bf7bdff69e0834debca Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 16:46:02 +0200 Subject: [PATCH 17/50] Add tests for RequestProcessor --- build.gradle.kts | 1 + .../processor/services/RequestProcessor.kt | 20 +- .../services/RequestProcessorTest.kt | 209 ++++++++++++++++++ 3 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 89df274..78bad77 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,6 +62,7 @@ dependencies { providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") + testImplementation("org.mockito.kotlin:mockito-kotlin:5.0.0") integrationTestImplementation("org.testcontainers:junit-jupiter") integrationTestImplementation("org.testcontainers:postgresql") } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index e04e568..fcb0863 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -49,15 +49,7 @@ class RequestProcessor( val pid = mtbFile.patient.id val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) - val allRequests = requestService.allRequestsByPatientPseudonym(pseudonymized.patient.id) - - val lastMtbFileRequestForPatient = allRequests - .filter { it.type == RequestType.MTB_FILE } - .firstOrNull { it.status == RequestStatus.SUCCESS || it.status == RequestStatus.WARNING } - - val isLastRequestDeletion = allRequests.firstOrNull()?.type == RequestType.DELETE - - if (null != lastMtbFileRequestForPatient && lastMtbFileRequestForPatient.fingerprint == fingerprint(mtbFile) && !isLastRequestDeletion) { + if (isDuplication(pseudonymized)) { requestService.save( Request( patientId = pseudonymized.patient.id, @@ -124,6 +116,16 @@ class RequestProcessor( statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) } + private fun isDuplication(pseudonymizedMtbFile: MtbFile): Boolean { + val lastMtbFileRequestForPatient = + requestService.lastMtbFileRequestForPatientPseudonym(pseudonymizedMtbFile.patient.id) + val isLastRequestDeletion = requestService.isLastRequestDeletion(pseudonymizedMtbFile.patient.id) + + return null != lastMtbFileRequestForPatient + && !isLastRequestDeletion + && lastMtbFileRequestForPatient.fingerprint == fingerprint(pseudonymizedMtbFile) + } + fun processDeletion(patientId: String) { val requestId = UUID.randomUUID().toString() diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt new file mode 100644 index 0000000..c165cf0 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -0,0 +1,209 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender +import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.Mockito.* +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import reactor.core.publisher.Sinks +import java.time.Instant +import java.util.* + + +@ExtendWith(MockitoExtension::class) +class RequestProcessorTest { + + private lateinit var pseudonymizeService: PseudonymizeService + private lateinit var sender: MtbFileSender + private lateinit var requestService: RequestService + private lateinit var statisticsUpdateProducer: Sinks.Many + + private lateinit var requestProcessor: RequestProcessor + + @BeforeEach + fun setup( + @Mock pseudonymizeService: PseudonymizeService, + @Mock sender: RestMtbFileSender, + @Mock requestService: RequestService, + ) { + this.pseudonymizeService = pseudonymizeService + this.sender = sender + this.requestService = requestService + this.statisticsUpdateProducer = Sinks.many().multicast().directBestEffort() + + val objectMapper = ObjectMapper() + + requestProcessor = RequestProcessor( + pseudonymizeService, + listOf(sender), + requestService, + objectMapper, + statisticsUpdateProducer + ) + } + + @Test + fun testShouldDetectMtbFileDuplicationAndSaveRequestStatus() { + doAnswer { + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "cwaxsvectyfj4qcw4hiwzx5fwwo7lekyagpzd2ayuf36jlvi6msa", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + }.`when`(requestService).lastMtbFileRequestForPatientPseudonym(anyString()) + + doAnswer { + false + }.`when`(requestService).isLastRequestDeletion(anyString()) + + doAnswer { + it.arguments[0] as MtbFile + }.`when`(pseudonymizeService).pseudonymize(any()) + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.DUPLICATION) + } + + @Test + fun testShouldSendMtbFileAndSaveRequestStatus() { + doAnswer { + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "different", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + }.`when`(requestService).lastMtbFileRequestForPatientPseudonym(anyString()) + + doAnswer { + false + }.`when`(requestService).isLastRequestDeletion(anyString()) + + doAnswer { + MtbFileSender.Response(status = MtbFileSender.ResponseStatus.SUCCESS) + }.`when`(sender).send(any()) + + doAnswer { + it.arguments[0] as MtbFile + }.`when`(pseudonymizeService).pseudonymize(any()) + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + + @Test + fun testShouldSendDeleterequestAndSaveRequestStatus() { + doAnswer { + "PSEUDONYM" + }.`when`(pseudonymizeService).patientPseudonym(anyString()) + + doAnswer { + MtbFileSender.Response(status = MtbFileSender.ResponseStatus.SUCCESS) + }.`when`(sender).send(any()) + + this.requestProcessor.processDeletion("TEST_12345678901") + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + +} \ No newline at end of file From 536ecbbd56d2dad166e995256a6793a675dea167 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 18:52:03 +0200 Subject: [PATCH 18/50] Add tests for error response status --- .../services/RequestProcessorTest.kt | 93 ++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index c165cf0..12d6e29 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -130,7 +130,7 @@ class RequestProcessorTest { } @Test - fun testShouldSendMtbFileAndSaveRequestStatus() { + fun testShouldSendMtbFileAndSaveSuccessRequestStatus() { doAnswer { Request( id = 1L, @@ -189,7 +189,66 @@ class RequestProcessorTest { } @Test - fun testShouldSendDeleterequestAndSaveRequestStatus() { + fun testShouldSendMtbFileAndSaveErrorRequestStatus() { + doAnswer { + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "different", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + }.`when`(requestService).lastMtbFileRequestForPatientPseudonym(anyString()) + + doAnswer { + false + }.`when`(requestService).isLastRequestDeletion(anyString()) + + doAnswer { + MtbFileSender.Response(status = MtbFileSender.ResponseStatus.ERROR) + }.`when`(sender).send(any()) + + doAnswer { + it.arguments[0] as MtbFile + }.`when`(pseudonymizeService).pseudonymize(any()) + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) + } + + @Test + fun testShouldSendDeleteRequestAndSaveSuccessRequestStatus() { doAnswer { "PSEUDONYM" }.`when`(pseudonymizeService).patientPseudonym(anyString()) @@ -206,4 +265,34 @@ class RequestProcessorTest { assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) } + @Test + fun testShouldSendDeleteRequestAndSaveErrorRequestStatus() { + doAnswer { + "PSEUDONYM" + }.`when`(pseudonymizeService).patientPseudonym(anyString()) + + doAnswer { + MtbFileSender.Response(status = MtbFileSender.ResponseStatus.ERROR) + }.`when`(sender).send(any()) + + this.requestProcessor.processDeletion("TEST_12345678901") + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) + } + + @Test + fun testShouldSendDeleteRequestWithPseudonymErrorAndSaveErrorRequestStatus() { + doThrow(RuntimeException()).`when`(pseudonymizeService).patientPseudonym(anyString()) + + this.requestProcessor.processDeletion("TEST_12345678901") + + val requestCaptor = argumentCaptor() + verify(requestService, times(1)).save(requestCaptor.capture()) + assertThat(requestCaptor.firstValue).isNotNull + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) + } + } \ No newline at end of file From 6ad6ee13a1cae8ed286e80b3a46c458e1052480b Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 8 Aug 2023 19:16:59 +0200 Subject: [PATCH 19/50] Ignore unknown properties in DataQualityResponse --- .../etl/processor/monitoring/ReportService.kt | 3 + .../processor/services/ReportServiceTest.kt | 70 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index 8c31ede..ae36705 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.monitoring +import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.databind.JsonMappingException @@ -46,8 +47,10 @@ class ReportService( } + @JsonIgnoreProperties(ignoreUnknown = true) private data class DataQualityReport(val issues: List) + @JsonIgnoreProperties(ignoreUnknown = true) data class Issue(val severity: Severity, val message: String) enum class Severity(@JsonValue val value: String) { diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt new file mode 100644 index 0000000..70efe2b --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt @@ -0,0 +1,70 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import dev.dnpm.etl.processor.monitoring.ReportService +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class ReportServiceTest { + + private lateinit var reportService: ReportService + + @BeforeEach + fun setup() { + this.reportService = ReportService(ObjectMapper().registerModule(KotlinModule.Builder().build())) + } + + @Test + fun shouldParseDataQualityReport() { + val json = """ + { + "patient": "4711", + "issues": [ + { "severity": "warning", "message": "Warning Message" }, + { "severity": "error", "message": "Error Message" } + ] + } + """.trimIndent() + + val actual = this.reportService.deserialize(json) + + assertThat(actual).hasSize(2) + assertThat(actual[0].severity).isEqualTo(ReportService.Severity.WARNING) + assertThat(actual[0].message).isEqualTo("Warning Message") + assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR) + assertThat(actual[1].message).isEqualTo("Error Message") + } + + @Test + fun shouldReturnSyntheticDataQualityReportOnParserError() { + val invalidResponse = "Invalid Response Data" + + val actual = this.reportService.deserialize(invalidResponse) + + assertThat(actual).hasSize(1) + assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR) + assertThat(actual[0].message).isEqualTo("Not parsable data quality report '$invalidResponse'") + } + +} \ No newline at end of file From 7739afad1fc82f4ffe0debbebae58874f046d82d Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 08:13:27 +0200 Subject: [PATCH 20/50] Handle MTB File with rejected consent as deletion request --- ...Controller.kt => MtbFileRestController.kt} | 14 +- .../web/MtbFileRestControllerTest.kt | 150 ++++++++++++++++++ 2 files changed, 160 insertions(+), 4 deletions(-) rename src/main/kotlin/dev/dnpm/etl/processor/web/{MtbFileController.kt => MtbFileRestController.kt} (77%) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/web/MtbFileRestControllerTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileRestController.kt similarity index 77% rename from src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt rename to src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileRestController.kt index cf0e693..9b441f6 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileRestController.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.web +import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile import dev.dnpm.etl.processor.services.RequestProcessor import org.slf4j.LoggerFactory @@ -26,16 +27,21 @@ import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.* @RestController -class MtbFileController( +class MtbFileRestController( private val requestProcessor: RequestProcessor, ) { - private val logger = LoggerFactory.getLogger(MtbFileController::class.java) + private val logger = LoggerFactory.getLogger(MtbFileRestController::class.java) @PostMapping(path = ["/mtbfile"]) fun mtbFile(@RequestBody mtbFile: MtbFile): ResponseEntity { - logger.debug("Accepted MTB File for processing") - requestProcessor.processMtbFile(mtbFile) + if (mtbFile.consent.status == Consent.Status.ACTIVE) { + logger.debug("Accepted MTB File for processing") + requestProcessor.processMtbFile(mtbFile) + } else { + logger.debug("Accepted MTB File and process deletion") + requestProcessor.processDeletion(mtbFile.patient.id) + } return ResponseEntity.accepted().build() } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/web/MtbFileRestControllerTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/web/MtbFileRestControllerTest.kt new file mode 100644 index 0000000..2fde35a --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/web/MtbFileRestControllerTest.kt @@ -0,0 +1,150 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.web + +import com.fasterxml.jackson.databind.ObjectMapper +import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.services.RequestProcessor +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.Mockito.times +import org.mockito.Mockito.verify +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.springframework.http.MediaType +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.delete +import org.springframework.test.web.servlet.post +import org.springframework.test.web.servlet.setup.MockMvcBuilders + +@ExtendWith(MockitoExtension::class) +class MtbFileRestControllerTest { + + private lateinit var mockMvc: MockMvc + + private lateinit var requestProcessor: RequestProcessor + + private val objectMapper = ObjectMapper() + + @BeforeEach + fun setup( + @Mock requestProcessor: RequestProcessor + ) { + this.requestProcessor = requestProcessor + val controller = MtbFileRestController(requestProcessor) + this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build() + } + + @Test + fun shouldProcessMtbFilePostRequest() { + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("TEST_12345678") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("TEST_12345678") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("TEST_12345678") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + mockMvc.post("/mtbfile") { + content = objectMapper.writeValueAsString(mtbFile) + contentType = MediaType.APPLICATION_JSON + }.andExpect { + status { + isAccepted() + } + } + + verify(requestProcessor, times(1)).processMtbFile(any()) + } + + @Test + fun shouldProcessMtbFilePostRequestWithRejectedConsent() { + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("TEST_12345678") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.REJECTED) + .withPatient("TEST_12345678") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("TEST_12345678") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + mockMvc.post("/mtbfile") { + content = objectMapper.writeValueAsString(mtbFile) + contentType = MediaType.APPLICATION_JSON + }.andExpect { + status { + isAccepted() + } + } + + val captor = argumentCaptor() + verify(requestProcessor, times(1)).processDeletion(captor.capture()) + assertThat(captor.firstValue).isEqualTo("TEST_12345678") + } + + @Test + fun shouldProcessMtbFileDeleteRequest() { + mockMvc.delete("/mtbfile/TEST_12345678").andExpect { + status { + isAccepted() + } + } + + val captor = argumentCaptor() + verify(requestProcessor, times(1)).processDeletion(captor.capture()) + assertThat(captor.firstValue).isEqualTo("TEST_12345678") + } + +} \ No newline at end of file From 13bfa0018d6c9b48893ef96945659be9e7eec6c0 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 10:20:20 +0200 Subject: [PATCH 21/50] Change endpoint configuration to select single endpoint * If REST endpoint is configured, it will be used * If Kafka endpoint is configured, it will be used * If both endpoints are configured, REST configuration has precedence and will be used --- README.md | 5 +- .../processor/config/AppConfigurationTest.kt | 102 ++++++++++++++++++ .../etl/processor/config/AppConfiguration.kt | 12 +-- .../processor/config/AppKafkaConfiguration.kt | 8 ++ .../processor/config/AppRestConfiguration.kt | 52 +++++++++ 5 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt diff --git a/README.md b/README.md index be6d6b5..ea0c02b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,10 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri ## Mögliche Endpunkte -Für REST-Requests als auch (parallel) zur Nutzung von Kafka-Topics können Endpunkte konfiguriert werden. +Für REST-Requests als auch zur Nutzung von Kafka-Topics können Endpunkte konfiguriert werden. + +Es ist dabei nur die Konfiguration eines Endpunkts zulässig. +Werden sowohl REST als auch Kafka-Endpunkt konfiguriert, wird nur der REST-Endpunkt verwendet. ### REST diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt new file mode 100644 index 0000000..8bdaa60 --- /dev/null +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt @@ -0,0 +1,102 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.config + +import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.output.KafkaMtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.springframework.beans.factory.NoSuchBeanDefinitionException +import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.boot.test.mock.mockito.MockBeans +import org.springframework.context.ApplicationContext +import org.springframework.test.context.ContextConfiguration +import org.springframework.test.context.TestPropertySource + +@SpringBootTest +@ContextConfiguration(classes = [KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class]) +class AppConfigurationTest { + + @Nested + @TestPropertySource( + properties = [ + "app.rest.uri=http://localhost:9000" + ] + ) + inner class AppConfigurationRestTest(private val context: ApplicationContext) { + + @Test + fun shouldUseRestMtbFileSenderNotKafkaMtbFileSender() { + assertThat(context.getBean(RestMtbFileSender::class.java)).isNotNull + assertThrows { context.getBean(KafkaMtbFileSender::class.java) } + } + + } + + @Nested + @TestPropertySource( + properties = [ + "app.kafka.servers=localhost:9092", + "app.kafka.topic=test", + "app.kafka.response-topic=test-response", + "app.kafka.group-id=test" + ] + ) + @MockBeans(value = [ + MockBean(ObjectMapper::class), + MockBean(RequestRepository::class) + ]) + inner class AppConfigurationKafkaTest(private val context: ApplicationContext) { + + @Test + fun shouldUseKafkaMtbFileSenderNotRestMtbFileSender() { + assertThrows { context.getBean(RestMtbFileSender::class.java) } + assertThat(context.getBean(KafkaMtbFileSender::class.java)).isNotNull + } + + } + + @Nested + @TestPropertySource( + properties = [ + "app.rest.uri=http://localhost:9000", + "app.kafka.servers=localhost:9092", + "app.kafka.topic=test", + "app.kafka.response-topic=test-response", + "app.kafka.group-id=test" + ] + ) + inner class AppConfigurationRestInPrecedenceTest(private val context: ApplicationContext) { + + @Test + fun shouldUseRestMtbFileSenderNotKafkaMtbFileSender() { + assertThat(context.getBean(RestMtbFileSender::class.java)).isNotNull + assertThrows { context.getBean(KafkaMtbFileSender::class.java) } + } + + } + +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index cbba1f1..6b15fc0 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -21,8 +21,6 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper import dev.dnpm.etl.processor.monitoring.ReportService -import dev.dnpm.etl.processor.output.MtbFileSender -import dev.dnpm.etl.processor.output.RestMtbFileSender import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator @@ -38,9 +36,7 @@ import reactor.core.publisher.Sinks value = [ AppConfigProperties::class, PseudonymizeConfigProperties::class, - GPasConfigProperties::class, - RestTargetProperties::class, - KafkaTargetProperties::class + GPasConfigProperties::class ] ) class AppConfiguration { @@ -65,12 +61,6 @@ class AppConfiguration { return PseudonymizeService(generator, pseudonymizeConfigProperties) } - @ConditionalOnProperty(value = ["app.rest.uri"]) - @Bean - fun restMtbFileSender(restTargetProperties: RestTargetProperties): MtbFileSender { - return RestMtbFileSender(restTargetProperties) - } - @Bean fun reportService(objectMapper: ObjectMapper): ReportService { return ReportService(objectMapper) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt index 7adcb02..6d0254e 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt @@ -24,10 +24,13 @@ import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor +import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.core.annotation.Order import org.springframework.kafka.core.ConsumerFactory import org.springframework.kafka.core.KafkaTemplate import org.springframework.kafka.listener.ContainerProperties @@ -38,14 +41,19 @@ import org.springframework.kafka.listener.KafkaMessageListenerContainer value = [KafkaTargetProperties::class] ) @ConditionalOnProperty(value = ["app.kafka.topic", "app.kafka.servers"]) +@ConditionalOnMissingBean(MtbFileSender::class) +@Order(-5) class AppKafkaConfiguration { + private val logger = LoggerFactory.getLogger(AppKafkaConfiguration::class.java) + @Bean fun kafkaMtbFileSender( kafkaTemplate: KafkaTemplate, kafkaTargetProperties: KafkaTargetProperties, objectMapper: ObjectMapper ): MtbFileSender { + logger.info("Selected 'KafkaMtbFileSender'") return KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, objectMapper) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt new file mode 100644 index 0000000..5e77a4f --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt @@ -0,0 +1,52 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.config + +import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender +import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.annotation.Order + +@Configuration +@EnableConfigurationProperties( + value = [ + RestTargetProperties::class + ] +) +@ConditionalOnProperty(value = ["app.rest.uri"]) +@ConditionalOnMissingBean(MtbFileSender::class) +@Order(-10) +class AppRestConfiguration { + + private val logger = LoggerFactory.getLogger(AppRestConfiguration::class.java) + + @Bean + fun restMtbFileSender(restTargetProperties: RestTargetProperties): MtbFileSender { + logger.info("Selected 'RestMtbFileSender'") + return RestMtbFileSender(restTargetProperties) + } + +} + From 47830ed9f7774c84674e9399cd347d12424f4f42 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 10:34:23 +0200 Subject: [PATCH 22/50] Use single MtbFileSender --- .../processor/EtlProcessorApplicationTests.kt | 5 +- .../services/RequestServiceIntegrationTest.kt | 3 + .../processor/services/RequestProcessor.kt | 114 ++++++++---------- .../services/RequestProcessorTest.kt | 2 +- 4 files changed, 59 insertions(+), 65 deletions(-) diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt index 07a201b..6c5b150 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt @@ -19,19 +19,22 @@ package dev.dnpm.etl.processor +import dev.dnpm.etl.processor.output.MtbFileSender import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.test.context.junit.jupiter.SpringExtension import org.testcontainers.junit.jupiter.Testcontainers @Testcontainers @ExtendWith(SpringExtension::class) @SpringBootTest +@MockBean(MtbFileSender::class) class EtlProcessorApplicationTests : AbstractTestcontainerTest() { @Test - fun contextLoads() { + fun contextLoadsIfMtbFileSenderConfigured() { } } diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt index d71e011..3af218e 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt @@ -24,12 +24,14 @@ import dev.dnpm.etl.processor.monitoring.Request import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.output.MtbFileSender import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.test.context.junit.jupiter.SpringExtension import org.springframework.transaction.annotation.Transactional import org.testcontainers.junit.jupiter.Testcontainers @@ -40,6 +42,7 @@ import java.util.* @ExtendWith(SpringExtension::class) @SpringBootTest @Transactional +@MockBean(MtbFileSender::class) class RequestServiceIntegrationTest : AbstractTestcontainerTest() { private lateinit var requestRepository: RequestRepository diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index fcb0863..936c1bf 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -37,7 +37,7 @@ import java.util.* @Service class RequestProcessor( private val pseudonymizeService: PseudonymizeService, - private val senders: List, + private val sender: MtbFileSender, private val requestService: RequestService, private val objectMapper: ObjectMapper, private val statisticsUpdateProducer: Sinks.Many @@ -66,32 +66,26 @@ class RequestProcessor( val request = MtbFileSender.MtbFileRequest(UUID.randomUUID().toString(), pseudonymized) - val responses = senders.map { - val responseStatus = it.send(request) - if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) { - logger.info( - "Sent file for Patient '{}' using '{}'", - pseudonymized.patient.id, - it.javaClass.simpleName - ) - } else { - logger.error( - "Error sending file for Patient '{}' using '{}'", - pseudonymized.patient.id, - it.javaClass.simpleName - ) - } - responseStatus + val responseStatus = sender.send(request) + if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) { + logger.info( + "Sent file for Patient '{}' using '{}'", + pseudonymized.patient.id, + sender.javaClass.simpleName + ) + } else { + logger.error( + "Error sending file for Patient '{}' using '{}'", + pseudonymized.patient.id, + sender.javaClass.simpleName + ) } - val requestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) { - RequestStatus.ERROR - } else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.WARNING)) { - RequestStatus.WARNING - } else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.SUCCESS)) { - RequestStatus.SUCCESS - } else { - RequestStatus.UNKNOWN + val requestStatus = when (responseStatus.status) { + MtbFileSender.ResponseStatus.ERROR -> RequestStatus.ERROR + MtbFileSender.ResponseStatus.WARNING -> RequestStatus.WARNING + MtbFileSender.ResponseStatus.SUCCESS -> RequestStatus.SUCCESS + else -> RequestStatus.UNKNOWN } requestService.save( @@ -104,9 +98,7 @@ class RequestProcessor( type = RequestType.MTB_FILE, report = when (requestStatus) { RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar") - RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten", - responses.joinToString("\n") { it.reason }) - + RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten", responseStatus.reason) RequestStatus.UNKNOWN -> Report("Keine Informationen") else -> null } @@ -132,42 +124,38 @@ class RequestProcessor( try { val patientPseudonym = pseudonymizeService.patientPseudonym(patientId) - val responses = senders.map { - val responseStatus = it.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym)) - when (responseStatus.status) { - MtbFileSender.ResponseStatus.SUCCESS -> { - logger.info( - "Sent delete for Patient '{}' using '{}'", - patientPseudonym, - it.javaClass.simpleName - ) - } - - MtbFileSender.ResponseStatus.ERROR -> { - logger.error( - "Error deleting data for Patient '{}' using '{}'", - patientPseudonym, - it.javaClass.simpleName - ) - } - - else -> { - logger.error( - "Unknown result on deleting data for Patient '{}' using '{}'", - patientPseudonym, - it.javaClass.simpleName - ) - } + val responseStatus = sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym)) + when (responseStatus.status) { + MtbFileSender.ResponseStatus.SUCCESS -> { + logger.info( + "Sent delete for Patient '{}' using '{}'", + patientPseudonym, + sender.javaClass.simpleName + ) + } + + MtbFileSender.ResponseStatus.ERROR -> { + logger.error( + "Error deleting data for Patient '{}' using '{}'", + patientPseudonym, + sender.javaClass.simpleName + ) + } + + else -> { + logger.error( + "Unknown result on deleting data for Patient '{}' using '{}'", + patientPseudonym, + sender.javaClass.simpleName + ) } - responseStatus } - val overallRequestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) { - RequestStatus.ERROR - } else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.SUCCESS)) { - RequestStatus.SUCCESS - } else { - RequestStatus.UNKNOWN + val requestStatus = when (responseStatus.status) { + MtbFileSender.ResponseStatus.ERROR -> RequestStatus.ERROR + MtbFileSender.ResponseStatus.WARNING -> RequestStatus.WARNING + MtbFileSender.ResponseStatus.SUCCESS -> RequestStatus.SUCCESS + else -> RequestStatus.UNKNOWN } requestService.save( @@ -176,9 +164,9 @@ class RequestProcessor( patientId = patientPseudonym, pid = patientId, fingerprint = fingerprint(patientPseudonym), - status = overallRequestStatus, + status = requestStatus, type = RequestType.DELETE, - report = when (overallRequestStatus) { + report = when (requestStatus) { RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar") RequestStatus.UNKNOWN -> Report("Keine Informationen") else -> null diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 12d6e29..6e97343 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -67,7 +67,7 @@ class RequestProcessorTest { requestProcessor = RequestProcessor( pseudonymizeService, - listOf(sender), + sender, requestService, objectMapper, statisticsUpdateProducer From 7f048e2483138deecc28208af42546097ef929d7 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 12:26:57 +0200 Subject: [PATCH 23/50] Do not append custom prefix to gPAS pseudonym --- .../pseudonym/PseudonymizeService.kt | 36 +------- .../etl/processor/pseudonym/extensions.kt | 50 +++++++++++ .../processor/services/RequestProcessor.kt | 14 +-- .../pseudonym/PseudonymizeServiceTest.kt | 86 +++++++++++++++++++ .../services/RequestProcessorTest.kt | 14 +-- 5 files changed, 155 insertions(+), 45 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeServiceTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt index 1a79850..ab8ce2f 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt @@ -19,7 +19,6 @@ package dev.dnpm.etl.processor.pseudonym -import de.ukw.ccc.bwhc.dto.MtbFile import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties class PseudonymizeService( @@ -27,38 +26,11 @@ class PseudonymizeService( private val configProperties: PseudonymizeConfigProperties ) { - fun pseudonymize(mtbFile: MtbFile): MtbFile { - val patientPseudonym = patientPseudonym(mtbFile.patient.id) - - mtbFile.episode.patient = patientPseudonym - mtbFile.carePlans.forEach { it.patient = patientPseudonym } - mtbFile.patient.id = patientPseudonym - mtbFile.claims.forEach { it.patient = patientPseudonym } - mtbFile.consent.patient = patientPseudonym - mtbFile.claimResponses.forEach { it.patient = patientPseudonym } - mtbFile.diagnoses.forEach { it.patient = patientPseudonym } - mtbFile.ecogStatus.forEach { it.patient = patientPseudonym } - mtbFile.familyMemberDiagnoses.forEach { it.patient = patientPseudonym } - mtbFile.geneticCounsellingRequests.forEach { it.patient = patientPseudonym } - mtbFile.histologyReevaluationRequests.forEach { it.patient = patientPseudonym } - mtbFile.histologyReports.forEach { it.patient = patientPseudonym } - mtbFile.lastGuidelineTherapies.forEach { it.patient = patientPseudonym } - mtbFile.molecularPathologyFindings.forEach { it.patient = patientPseudonym } - mtbFile.molecularTherapies.forEach { it.history.forEach { it.patient = patientPseudonym } } - mtbFile.ngsReports.forEach { it.patient = patientPseudonym } - mtbFile.previousGuidelineTherapies.forEach { it.patient = patientPseudonym } - mtbFile.rebiopsyRequests.forEach { it.patient = patientPseudonym } - mtbFile.recommendations.forEach { it.patient = patientPseudonym } - mtbFile.recommendations.forEach { it.patient = patientPseudonym } - mtbFile.responses.forEach { it.patient = patientPseudonym } - mtbFile.specimens.forEach { it.patient = patientPseudonym } - mtbFile.specimens.forEach { it.patient = patientPseudonym } - - return mtbFile - } - fun patientPseudonym(patientId: String): String { - return "${configProperties.prefix}_${generator.generate(patientId)}" + return when (generator) { + is GpasPseudonymGenerator -> generator.generate(patientId) + else -> "${configProperties.prefix}_${generator.generate(patientId)}" + } } } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt new file mode 100644 index 0000000..580785d --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -0,0 +1,50 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.pseudonym + +import de.ukw.ccc.bwhc.dto.MtbFile + +infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) { + val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id) + + this.episode.patient = patientPseudonym + this.carePlans.forEach { it.patient = patientPseudonym } + this.patient.id = patientPseudonym + this.claims.forEach { it.patient = patientPseudonym } + this.consent.patient = patientPseudonym + this.claimResponses.forEach { it.patient = patientPseudonym } + this.diagnoses.forEach { it.patient = patientPseudonym } + this.ecogStatus.forEach { it.patient = patientPseudonym } + this.familyMemberDiagnoses.forEach { it.patient = patientPseudonym } + this.geneticCounsellingRequests.forEach { it.patient = patientPseudonym } + this.histologyReevaluationRequests.forEach { it.patient = patientPseudonym } + this.histologyReports.forEach { it.patient = patientPseudonym } + this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym } + this.molecularPathologyFindings.forEach { it.patient = patientPseudonym } + this.molecularTherapies.forEach { it.history.forEach { it.patient = patientPseudonym } } + this.ngsReports.forEach { it.patient = patientPseudonym } + this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym } + this.rebiopsyRequests.forEach { it.patient = patientPseudonym } + this.recommendations.forEach { it.patient = patientPseudonym } + this.recommendations.forEach { it.patient = patientPseudonym } + this.responses.forEach { it.patient = patientPseudonym } + this.specimens.forEach { it.patient = patientPseudonym } + this.specimens.forEach { it.patient = patientPseudonym } +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 936c1bf..6465e82 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -27,6 +27,7 @@ import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.monitoring.RequestType import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith import org.apache.commons.codec.binary.Base32 import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory @@ -47,12 +48,13 @@ class RequestProcessor( fun processMtbFile(mtbFile: MtbFile) { val pid = mtbFile.patient.id - val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) - if (isDuplication(pseudonymized)) { + mtbFile pseudonymizeWith pseudonymizeService + + if (isDuplication(mtbFile)) { requestService.save( Request( - patientId = pseudonymized.patient.id, + patientId = mtbFile.patient.id, pid = pid, fingerprint = fingerprint(mtbFile), status = RequestStatus.DUPLICATION, @@ -64,19 +66,19 @@ class RequestProcessor( return } - val request = MtbFileSender.MtbFileRequest(UUID.randomUUID().toString(), pseudonymized) + val request = MtbFileSender.MtbFileRequest(UUID.randomUUID().toString(), mtbFile) val responseStatus = sender.send(request) if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) { logger.info( "Sent file for Patient '{}' using '{}'", - pseudonymized.patient.id, + mtbFile.patient.id, sender.javaClass.simpleName ) } else { logger.error( "Error sending file for Patient '{}' using '{}'", - pseudonymized.patient.id, + mtbFile.patient.id, sender.javaClass.simpleName ) } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeServiceTest.kt new file mode 100644 index 0000000..a30a328 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeServiceTest.kt @@ -0,0 +1,86 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.pseudonym + +import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.whenever + +@ExtendWith(MockitoExtension::class) +class PseudonymizeServiceTest { + + private val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("123") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("123") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + @Test + fun shouldNotUsePseudonymPrefixForGpas(@Mock generator: GpasPseudonymGenerator) { + doAnswer { + it.arguments[0] + }.whenever(generator).generate(anyString()) + + val pseudonymizeService = PseudonymizeService(generator, PseudonymizeConfigProperties()) + + mtbFile.pseudonymizeWith(pseudonymizeService) + + assertThat(mtbFile.patient.id).isEqualTo("123") + } + + @Test + fun shouldUsePseudonymPrefixForBuiltin(@Mock generator: AnonymizingGenerator) { + doAnswer { + it.arguments[0] + }.whenever(generator).generate(anyString()) + + val pseudonymizeService = PseudonymizeService(generator, PseudonymizeConfigProperties()) + + mtbFile.pseudonymizeWith(pseudonymizeService) + + assertThat(mtbFile.patient.id).isEqualTo("UNKNOWN_123") + } + +} \ No newline at end of file diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 6e97343..8552bbb 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -82,7 +82,7 @@ class RequestProcessorTest { uuid = UUID.randomUUID().toString(), patientId = "TEST_12345678901", pid = "P1", - fingerprint = "cwaxsvectyfj4qcw4hiwzx5fwwo7lekyagpzd2ayuf36jlvi6msa", + fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a", type = RequestType.MTB_FILE, status = RequestStatus.SUCCESS, processedAt = Instant.parse("2023-08-08T02:00:00Z") @@ -94,8 +94,8 @@ class RequestProcessorTest { }.`when`(requestService).isLastRequestDeletion(anyString()) doAnswer { - it.arguments[0] as MtbFile - }.`when`(pseudonymizeService).pseudonymize(any()) + it.arguments[0] as String + }.`when`(pseudonymizeService).patientPseudonym(any()) val mtbFile = MtbFile.builder() .withPatient( @@ -153,8 +153,8 @@ class RequestProcessorTest { }.`when`(sender).send(any()) doAnswer { - it.arguments[0] as MtbFile - }.`when`(pseudonymizeService).pseudonymize(any()) + it.arguments[0] as String + }.`when`(pseudonymizeService).patientPseudonym(any()) val mtbFile = MtbFile.builder() .withPatient( @@ -212,8 +212,8 @@ class RequestProcessorTest { }.`when`(sender).send(any()) doAnswer { - it.arguments[0] as MtbFile - }.`when`(pseudonymizeService).pseudonymize(any()) + it.arguments[0] as String + }.`when`(pseudonymizeService).patientPseudonym(any()) val mtbFile = MtbFile.builder() .withPatient( From 1a640ff9dff1cc182c4ffc1d00dff370e42a25de Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 18:15:20 +0200 Subject: [PATCH 24/50] Decouple request and response processing --- .../processor/config/AppKafkaConfiguration.kt | 6 +- .../processor/output/KafkaMtbFileSender.kt | 13 +- .../etl/processor/output/MtbFileSender.kt | 21 ++- .../etl/processor/output/RestMtbFileSender.kt | 16 +- .../processor/services/RequestProcessor.kt | 139 ++++++----------- .../processor/services/ResponseProcessor.kt | 96 ++++++++++++ .../services/kafka/KafkaResponseProcessor.kt | 93 ++++++------ .../services/RequestProcessorTest.kt | 128 ++++++++++++---- .../services/ResponseProcessorTest.kt | 142 ++++++++++++++++++ .../kafka/KafkaResponseProcessorTest.kt | 119 +++++++++++++++ 10 files changed, 579 insertions(+), 194 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt index 6d0254e..309ff2d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt @@ -20,7 +20,6 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor @@ -28,6 +27,7 @@ import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.ApplicationEventPublisher import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.annotation.Order @@ -70,10 +70,10 @@ class AppKafkaConfiguration { @Bean fun kafkaResponseProcessor( - requestRepository: RequestRepository, + applicationEventPublisher: ApplicationEventPublisher, objectMapper: ObjectMapper ): KafkaResponseProcessor { - return KafkaResponseProcessor(requestRepository, objectMapper) + return KafkaResponseProcessor(applicationEventPublisher, objectMapper) } } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index d903745..9448e29 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile import dev.dnpm.etl.processor.config.KafkaTargetProperties +import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate @@ -43,13 +44,13 @@ class KafkaMtbFileSender( ) if (result.get() != null) { logger.debug("Sent file via KafkaMtbFileSender") - MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.UNKNOWN) } else { - MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) + MtbFileSender.Response(RequestStatus.ERROR) } } catch (e: Exception) { logger.error("An error occurred sending to kafka", e) - MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.UNKNOWN) } } @@ -72,13 +73,13 @@ class KafkaMtbFileSender( if (result.get() != null) { logger.debug("Sent deletion request via KafkaMtbFileSender") - MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.UNKNOWN) } else { - MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) + MtbFileSender.Response(RequestStatus.ERROR) } } catch (e: Exception) { logger.error("An error occurred sending to kafka", e) - MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.UNKNOWN) } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt index 6914ba5..de0efaa 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt @@ -20,22 +20,31 @@ package dev.dnpm.etl.processor.output import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.springframework.http.HttpStatusCode interface MtbFileSender { fun send(request: MtbFileRequest): Response fun send(request: DeleteRequest): Response - data class Response(val status: ResponseStatus, val reason: String = "") + data class Response(val status: RequestStatus, val body: String = "") data class MtbFileRequest(val requestId: String, val mtbFile: MtbFile) data class DeleteRequest(val requestId: String, val patientId: String) - enum class ResponseStatus { - SUCCESS, - WARNING, - ERROR, - UNKNOWN +} + +fun Int.asRequestStatus(): RequestStatus { + return when (this) { + 200 -> RequestStatus.SUCCESS + 201 -> RequestStatus.WARNING + in 400 .. 999 -> RequestStatus.ERROR + else -> RequestStatus.UNKNOWN } +} + +fun HttpStatusCode.asRequestStatus(): RequestStatus { + return this.value().asRequestStatus() } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt index 04c73ef..24cdc49 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -20,13 +20,13 @@ package dev.dnpm.etl.processor.output import dev.dnpm.etl.processor.config.RestTargetProperties +import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory import org.springframework.http.HttpEntity import org.springframework.http.HttpHeaders import org.springframework.http.MediaType import org.springframework.web.client.RestClientException import org.springframework.web.client.RestTemplate -import org.springframework.web.util.UriComponentsBuilder class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) : MtbFileSender { @@ -46,21 +46,17 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) ) if (!response.statusCode.is2xxSuccessful) { logger.warn("Error sending to remote system: {}", response.body) - return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Status-Code: ${response.statusCode.value()}") + return MtbFileSender.Response(response.statusCode.asRequestStatus(), "Status-Code: ${response.statusCode.value()}") } logger.debug("Sent file via RestMtbFileSender") - return if (response.body?.contains("warning") == true) { - MtbFileSender.Response(MtbFileSender.ResponseStatus.WARNING, "${response.body}") - } else { - MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) - } + return MtbFileSender.Response(response.statusCode.asRequestStatus()) } catch (e: IllegalArgumentException) { logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) } catch (e: RestClientException) { logger.info(restTargetProperties.uri!!.toString()) logger.error("Cannot send data to remote system", e) } - return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Sonstiger Fehler bei der Übertragung") + return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") } override fun send(request: MtbFileSender.DeleteRequest): MtbFileSender.Response { @@ -74,14 +70,14 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) String::class.java ) logger.debug("Sent file via RestMtbFileSender") - return MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) + return MtbFileSender.Response(RequestStatus.SUCCESS) } catch (e: IllegalArgumentException) { logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) } catch (e: RestClientException) { logger.info(restTargetProperties.uri!!.toString()) logger.error("Cannot send data to remote system", e) } - return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Sonstiger Fehler bei der Übertragung") + return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") } } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 6465e82..d2f8619 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -31,8 +31,9 @@ import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith import org.apache.commons.codec.binary.Base32 import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory +import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Service -import reactor.core.publisher.Sinks +import java.time.Instant import java.util.* @Service @@ -41,73 +42,54 @@ class RequestProcessor( private val sender: MtbFileSender, private val requestService: RequestService, private val objectMapper: ObjectMapper, - private val statisticsUpdateProducer: Sinks.Many + private val applicationEventPublisher: ApplicationEventPublisher ) { private val logger = LoggerFactory.getLogger(RequestProcessor::class.java) fun processMtbFile(mtbFile: MtbFile) { + val requestId = UUID.randomUUID().toString() val pid = mtbFile.patient.id mtbFile pseudonymizeWith pseudonymizeService - if (isDuplication(mtbFile)) { - requestService.save( - Request( - patientId = mtbFile.patient.id, - pid = pid, - fingerprint = fingerprint(mtbFile), - status = RequestStatus.DUPLICATION, - type = RequestType.MTB_FILE, - report = Report("Duplikat erkannt - keine Daten weitergeleitet") - ) - ) - statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) - return - } - - val request = MtbFileSender.MtbFileRequest(UUID.randomUUID().toString(), mtbFile) - - val responseStatus = sender.send(request) - if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) { - logger.info( - "Sent file for Patient '{}' using '{}'", - mtbFile.patient.id, - sender.javaClass.simpleName - ) - } else { - logger.error( - "Error sending file for Patient '{}' using '{}'", - mtbFile.patient.id, - sender.javaClass.simpleName - ) - } - - val requestStatus = when (responseStatus.status) { - MtbFileSender.ResponseStatus.ERROR -> RequestStatus.ERROR - MtbFileSender.ResponseStatus.WARNING -> RequestStatus.WARNING - MtbFileSender.ResponseStatus.SUCCESS -> RequestStatus.SUCCESS - else -> RequestStatus.UNKNOWN - } + val request = MtbFileSender.MtbFileRequest(requestId, mtbFile) requestService.save( Request( - uuid = request.requestId, + uuid = requestId, patientId = request.mtbFile.patient.id, pid = pid, fingerprint = fingerprint(request.mtbFile), - status = requestStatus, - type = RequestType.MTB_FILE, - report = when (requestStatus) { - RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar") - RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten", responseStatus.reason) - RequestStatus.UNKNOWN -> Report("Keine Informationen") - else -> null - } + status = RequestStatus.UNKNOWN, + type = RequestType.MTB_FILE ) ) - statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) + if (isDuplication(mtbFile)) { + applicationEventPublisher.publishEvent( + ResponseEvent( + requestId, + Instant.now(), + RequestStatus.DUPLICATION + ) + ) + return + } + + val responseStatus = sender.send(request) + + applicationEventPublisher.publishEvent( + ResponseEvent( + requestId, + Instant.now(), + responseStatus.status, + when (responseStatus.status) { + RequestStatus.WARNING -> Optional.of(responseStatus.body) + else -> Optional.empty() + } + ) + ) } private fun isDuplication(pseudonymizedMtbFile: MtbFile): Boolean { @@ -126,55 +108,31 @@ class RequestProcessor( try { val patientPseudonym = pseudonymizeService.patientPseudonym(patientId) - val responseStatus = sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym)) - when (responseStatus.status) { - MtbFileSender.ResponseStatus.SUCCESS -> { - logger.info( - "Sent delete for Patient '{}' using '{}'", - patientPseudonym, - sender.javaClass.simpleName - ) - } - - MtbFileSender.ResponseStatus.ERROR -> { - logger.error( - "Error deleting data for Patient '{}' using '{}'", - patientPseudonym, - sender.javaClass.simpleName - ) - } - - else -> { - logger.error( - "Unknown result on deleting data for Patient '{}' using '{}'", - patientPseudonym, - sender.javaClass.simpleName - ) - } - } - - val requestStatus = when (responseStatus.status) { - MtbFileSender.ResponseStatus.ERROR -> RequestStatus.ERROR - MtbFileSender.ResponseStatus.WARNING -> RequestStatus.WARNING - MtbFileSender.ResponseStatus.SUCCESS -> RequestStatus.SUCCESS - else -> RequestStatus.UNKNOWN - } - requestService.save( Request( uuid = requestId, patientId = patientPseudonym, pid = patientId, fingerprint = fingerprint(patientPseudonym), - status = requestStatus, - type = RequestType.DELETE, - report = when (requestStatus) { - RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar") - RequestStatus.UNKNOWN -> Report("Keine Informationen") - else -> null + status = RequestStatus.UNKNOWN, + type = RequestType.DELETE + ) + ) + + val responseStatus = sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym)) + + applicationEventPublisher.publishEvent( + ResponseEvent( + requestId, + Instant.now(), + responseStatus.status, + when (responseStatus.status) { + RequestStatus.WARNING, RequestStatus.ERROR -> Optional.of(responseStatus.body) + else -> Optional.empty() } ) ) + } catch (e: Exception) { requestService.save( Request( @@ -188,7 +146,6 @@ class RequestProcessor( ) ) } - statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) } private fun fingerprint(mtbFile: MtbFile): String { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt new file mode 100644 index 0000000..d7ad86f --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -0,0 +1,96 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.monitoring.Report +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.slf4j.LoggerFactory +import org.springframework.context.event.EventListener +import org.springframework.stereotype.Service +import reactor.core.publisher.Sinks +import java.time.Instant +import java.util.* + +@Service +class ResponseProcessor( + private val requestRepository: RequestRepository, + private val statisticsUpdateProducer: Sinks.Many, + private val objectMapper: ObjectMapper +) { + + private val logger = LoggerFactory.getLogger(ResponseProcessor::class.java) + + @EventListener(classes = [ResponseEvent::class]) + fun handleResponseEvent(event: ResponseEvent) { + requestRepository.findByUuidEquals(event.requestUuid).ifPresentOrElse({ + it.processedAt = event.timestamp + it.status = event.status + + when (event.status) { + RequestStatus.SUCCESS -> { + it.report = Report( + "Keine Probleme erkannt", + ) + } + + RequestStatus.WARNING -> { + it.report = Report( + "Warnungen über mangelhafte Daten", + objectMapper.writeValueAsString(event.body) + ) + } + + RequestStatus.ERROR -> { + it.report = Report( + "Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar", + objectMapper.writeValueAsString(event.body) + ) + } + + RequestStatus.DUPLICATION -> { + it.report = Report( + "Duplikat erkannt" + ) + } + + else -> { + logger.error("Cannot process response: Unknown response code!") + return@ifPresentOrElse + } + } + + requestRepository.save(it) + + statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) + }, { + logger.error("Response for unknown request '${event.requestUuid}'!") + }) + } + +} + +data class ResponseEvent( + val requestUuid: String, + val timestamp: Instant, + val status: RequestStatus, + val body: Optional = Optional.empty() +) \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index 1e9263d..ef880f4 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -22,16 +22,18 @@ package dev.dnpm.etl.processor.services.kafka import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper -import dev.dnpm.etl.processor.monitoring.Report -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.output.asRequestStatus +import dev.dnpm.etl.processor.services.ResponseEvent import org.apache.kafka.clients.consumer.ConsumerRecord import org.slf4j.LoggerFactory +import org.springframework.context.ApplicationEventPublisher import org.springframework.kafka.listener.MessageListener import java.time.Instant +import java.util.* class KafkaResponseProcessor( - private val requestRepository: RequestRepository, + private val eventPublisher: ApplicationEventPublisher, private val objectMapper: ObjectMapper ) : MessageListener { @@ -39,55 +41,44 @@ class KafkaResponseProcessor( override fun onMessage(data: ConsumerRecord) { try { - val responseKey = objectMapper.readValue(data.key(), ResponseKey::class.java) - requestRepository.findByUuidEquals(responseKey.requestId).ifPresent { - val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) - - when (responseBody.statusCode) { - 200 -> { - it.status = RequestStatus.SUCCESS - it.processedAt = Instant.ofEpochMilli(data.timestamp()) - requestRepository.save(it) - } - - 201 -> { - it.status = RequestStatus.WARNING - it.processedAt = Instant.ofEpochMilli(data.timestamp()) - it.report = Report( - "Warnungen über mangelhafte Daten", - objectMapper.writeValueAsString(responseBody.statusBody) - ) - requestRepository.save(it) - } - - 400, 422 -> { - it.status = RequestStatus.ERROR - it.processedAt = Instant.ofEpochMilli(data.timestamp()) - it.report = Report( - "Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar", - objectMapper.writeValueAsString(responseBody.statusBody) - ) - requestRepository.save(it) - } - - in 900..999 -> { - it.status = RequestStatus.ERROR - it.processedAt = Instant.ofEpochMilli(data.timestamp()) - it.report = Report( - "Fehler bei der Datenübertragung, keine Verbindung zum bwHC-Backend möglich", - objectMapper.writeValueAsString(responseBody.statusBody) - ) - requestRepository.save(it) - } - - else -> { - logger.error("Cannot process Kafka response: Unknown response code!") - } - } - } + Optional.of(objectMapper.readValue(data.key(), ResponseKey::class.java)) } catch (e: Exception) { - logger.error("Cannot process Kafka response", e) - } + Optional.empty() + }.ifPresentOrElse({ responseKey -> + val event = try { + val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) + ResponseEvent( + responseKey.requestId, + Instant.ofEpochMilli(data.timestamp()), + responseBody.statusCode.asRequestStatus(), + when (responseBody.statusCode.asRequestStatus()) { + RequestStatus.SUCCESS -> { + Optional.empty() + } + + RequestStatus.WARNING, RequestStatus.ERROR -> { + Optional.of(objectMapper.writeValueAsString(responseBody.statusBody)) + } + + else -> { + logger.error("Kafka response: Unknown response code!") + Optional.empty() + } + } + ) + } catch (e: Exception) { + logger.error("Cannot process Kafka response", e) + ResponseEvent( + responseKey.requestId, + Instant.ofEpochMilli(data.timestamp()), + RequestStatus.ERROR, + Optional.of("Cannot process Kafka response") + ) + } + eventPublisher.publishEvent(event) + }, { + logger.error("No response key in Kafka response") + }) } data class ResponseKey(val requestId: String) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 8552bbb..f9d8182 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -37,7 +37,7 @@ import org.mockito.Mockito.* import org.mockito.junit.jupiter.MockitoExtension import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor -import reactor.core.publisher.Sinks +import org.springframework.context.ApplicationEventPublisher import java.time.Instant import java.util.* @@ -48,7 +48,7 @@ class RequestProcessorTest { private lateinit var pseudonymizeService: PseudonymizeService private lateinit var sender: MtbFileSender private lateinit var requestService: RequestService - private lateinit var statisticsUpdateProducer: Sinks.Many + private lateinit var applicationEventPublisher: ApplicationEventPublisher private lateinit var requestProcessor: RequestProcessor @@ -57,11 +57,12 @@ class RequestProcessorTest { @Mock pseudonymizeService: PseudonymizeService, @Mock sender: RestMtbFileSender, @Mock requestService: RequestService, + @Mock applicationEventPublisher: ApplicationEventPublisher ) { this.pseudonymizeService = pseudonymizeService this.sender = sender this.requestService = requestService - this.statisticsUpdateProducer = Sinks.many().multicast().directBestEffort() + this.applicationEventPublisher = applicationEventPublisher val objectMapper = ObjectMapper() @@ -70,12 +71,12 @@ class RequestProcessorTest { sender, requestService, objectMapper, - statisticsUpdateProducer + applicationEventPublisher ) } @Test - fun testShouldDetectMtbFileDuplicationAndSaveRequestStatus() { + fun testShouldSendMtbFileDuplicationAndSaveUnknownRequestStatusAtFirst() { doAnswer { Request( id = 1L, @@ -126,11 +127,66 @@ class RequestProcessorTest { val requestCaptor = argumentCaptor() verify(requestService, times(1)).save(requestCaptor.capture()) assertThat(requestCaptor.firstValue).isNotNull - assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.DUPLICATION) + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.UNKNOWN) } @Test - fun testShouldSendMtbFileAndSaveSuccessRequestStatus() { + fun testShouldDetectMtbFileDuplicationAndSendDuplicationEvent() { + doAnswer { + Request( + id = 1L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a", + type = RequestType.MTB_FILE, + status = RequestStatus.SUCCESS, + processedAt = Instant.parse("2023-08-08T02:00:00Z") + ) + }.`when`(requestService).lastMtbFileRequestForPatientPseudonym(anyString()) + + doAnswer { + false + }.`when`(requestService).isLastRequestDeletion(anyString()) + + doAnswer { + it.arguments[0] as String + }.`when`(pseudonymizeService).patientPseudonym(any()) + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.DUPLICATION) + } + + @Test + fun testShouldSendMtbFileAndSendSuccessEvent() { doAnswer { Request( id = 1L, @@ -149,7 +205,7 @@ class RequestProcessorTest { }.`when`(requestService).isLastRequestDeletion(anyString()) doAnswer { - MtbFileSender.Response(status = MtbFileSender.ResponseStatus.SUCCESS) + MtbFileSender.Response(status = RequestStatus.SUCCESS) }.`when`(sender).send(any()) doAnswer { @@ -182,14 +238,14 @@ class RequestProcessorTest { this.requestProcessor.processMtbFile(mtbFile) - val requestCaptor = argumentCaptor() - verify(requestService, times(1)).save(requestCaptor.capture()) - assertThat(requestCaptor.firstValue).isNotNull - assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) } @Test - fun testShouldSendMtbFileAndSaveErrorRequestStatus() { + fun testShouldSendMtbFileAndSendErrorEvent() { doAnswer { Request( id = 1L, @@ -208,7 +264,7 @@ class RequestProcessorTest { }.`when`(requestService).isLastRequestDeletion(anyString()) doAnswer { - MtbFileSender.Response(status = MtbFileSender.ResponseStatus.ERROR) + MtbFileSender.Response(status = RequestStatus.ERROR) }.`when`(sender).send(any()) doAnswer { @@ -241,20 +297,20 @@ class RequestProcessorTest { this.requestProcessor.processMtbFile(mtbFile) - val requestCaptor = argumentCaptor() - verify(requestService, times(1)).save(requestCaptor.capture()) - assertThat(requestCaptor.firstValue).isNotNull - assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) } @Test - fun testShouldSendDeleteRequestAndSaveSuccessRequestStatus() { + fun testShouldSendDeleteRequestAndSaveUnknownRequestStatusAtFirst() { doAnswer { "PSEUDONYM" }.`when`(pseudonymizeService).patientPseudonym(anyString()) doAnswer { - MtbFileSender.Response(status = MtbFileSender.ResponseStatus.SUCCESS) + MtbFileSender.Response(status = RequestStatus.UNKNOWN) }.`when`(sender).send(any()) this.requestProcessor.processDeletion("TEST_12345678901") @@ -262,25 +318,43 @@ class RequestProcessorTest { val requestCaptor = argumentCaptor() verify(requestService, times(1)).save(requestCaptor.capture()) assertThat(requestCaptor.firstValue).isNotNull - assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.UNKNOWN) } @Test - fun testShouldSendDeleteRequestAndSaveErrorRequestStatus() { + fun testShouldSendDeleteRequestAndSendSuccessEvent() { doAnswer { "PSEUDONYM" }.`when`(pseudonymizeService).patientPseudonym(anyString()) doAnswer { - MtbFileSender.Response(status = MtbFileSender.ResponseStatus.ERROR) + MtbFileSender.Response(status = RequestStatus.SUCCESS) }.`when`(sender).send(any()) this.requestProcessor.processDeletion("TEST_12345678901") - val requestCaptor = argumentCaptor() - verify(requestService, times(1)).save(requestCaptor.capture()) - assertThat(requestCaptor.firstValue).isNotNull - assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + + @Test + fun testShouldSendDeleteRequestAndSendErrorEvent() { + doAnswer { + "PSEUDONYM" + }.`when`(pseudonymizeService).patientPseudonym(anyString()) + + doAnswer { + MtbFileSender.Response(status = RequestStatus.ERROR) + }.`when`(sender).send(any()) + + this.requestProcessor.processDeletion("TEST_12345678901") + + val eventCaptor = argumentCaptor() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR) } @Test diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt new file mode 100644 index 0000000..cfb1111 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt @@ -0,0 +1,142 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import dev.dnpm.etl.processor.monitoring.Request +import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.RequestType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.* +import reactor.core.publisher.Sinks +import java.time.Instant +import java.util.* + +@ExtendWith(MockitoExtension::class) +class ResponseProcessorTest { + + private lateinit var requestRepository: RequestRepository + private lateinit var statisticsUpdateProducer: Sinks.Many + + private lateinit var responseProcessor: ResponseProcessor + + private val testRequest = Request( + 1L, + "TestID1234", + "PSEUDONYM-A", + "1", + "dummyfingerprint", + RequestType.MTB_FILE, + RequestStatus.UNKNOWN + ) + + @BeforeEach + fun setup( + @Mock requestRepository: RequestRepository, + @Mock statisticsUpdateProducer: Sinks.Many + ) { + val objectMapper = ObjectMapper().registerModule(KotlinModule.Builder().build()) + + this.requestRepository = requestRepository + this.statisticsUpdateProducer = statisticsUpdateProducer + + this.responseProcessor = ResponseProcessor(requestRepository, statisticsUpdateProducer, objectMapper) + } + + @Test + fun shouldNotSaveStatusForUnknownRequest() { + doAnswer { + Optional.empty() + }.whenever(requestRepository).findByUuidEquals(anyString()) + + val event = ResponseEvent( + "TestID1234", + Instant.parse("2023-09-09T00:00:00Z"), + RequestStatus.SUCCESS + ) + + this.responseProcessor.handleResponseEvent(event) + + verify(requestRepository, never()).save(any()) + } + + @Test + fun shouldNotSaveStatusWithUnknownState() { + doAnswer { + Optional.of(testRequest) + }.whenever(requestRepository).findByUuidEquals(anyString()) + + val event = ResponseEvent( + "TestID1234", + Instant.parse("2023-09-09T00:00:00Z"), + RequestStatus.UNKNOWN + ) + + this.responseProcessor.handleResponseEvent(event) + + verify(requestRepository, never()).save(any()) + } + + @ParameterizedTest + @MethodSource("requestStatusSource") + fun shouldSaveStatusForKnownRequest(requestStatus: RequestStatus) { + doAnswer { + Optional.of(testRequest) + }.whenever(requestRepository).findByUuidEquals(anyString()) + + val event = ResponseEvent( + "TestID1234", + Instant.parse("2023-09-09T00:00:00Z"), + requestStatus + ) + + this.responseProcessor.handleResponseEvent(event) + + val captor = argumentCaptor() + verify(requestRepository, times(1)).save(captor.capture()) + assertThat(captor.firstValue).isNotNull + assertThat(captor.firstValue.status).isEqualTo(requestStatus) + } + + companion object { + + @JvmStatic + fun requestStatusSource(): Set { + return setOf( + RequestStatus.SUCCESS, + RequestStatus.WARNING, + RequestStatus.ERROR, + RequestStatus.DUPLICATION + ) + } + + } + +} \ No newline at end of file diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt new file mode 100644 index 0000000..0f524ca --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt @@ -0,0 +1,119 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.services.kafka + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import dev.dnpm.etl.processor.services.ResponseEvent +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.any +import org.mockito.kotlin.never +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.springframework.context.ApplicationEventPublisher +import org.springframework.http.HttpStatus + +@ExtendWith(MockitoExtension::class) +class KafkaResponseProcessorTest { + + private lateinit var eventPublisher: ApplicationEventPublisher + private lateinit var objectMapper: ObjectMapper + + private lateinit var kafkaResponseProcessor: KafkaResponseProcessor + + private fun createkafkaRecord( + requestId: String? = null, + statusCode: Int = 200, + statusBody: Map? = mapOf() + ): ConsumerRecord { + return ConsumerRecord( + "test-topic", + 0, + 0, + if (requestId == null) { + null + } else { + this.objectMapper.writeValueAsString(KafkaResponseProcessor.ResponseKey(requestId)) + }, + if (statusBody == null) { + "" + } else { + this.objectMapper.writeValueAsString(KafkaResponseProcessor.ResponseBody(statusCode, statusBody)) + } + ) + } + + @BeforeEach + fun setup( + @Mock eventPublisher: ApplicationEventPublisher + ) { + this.eventPublisher = eventPublisher + this.objectMapper = ObjectMapper().registerModule(KotlinModule.Builder().build()) + + this.kafkaResponseProcessor = KafkaResponseProcessor(eventPublisher, objectMapper) + } + + @Test + fun shouldNotProcessRecordsWithoutValidKey() { + this.kafkaResponseProcessor.onMessage(createkafkaRecord(null, 200)) + + verify(eventPublisher, never()).publishEvent(any()) + } + + @Test + fun shouldNotProcessRecordsWithoutValidBody() { + this.kafkaResponseProcessor.onMessage(createkafkaRecord(requestId = "TestID1234", statusBody = null)) + + verify(eventPublisher, never()).publishEvent(any()) + } + + @ParameterizedTest + @MethodSource("statusCodeSource") + fun shouldProcessValidRecordsWithStatusCode(statusCode: Int) { + this.kafkaResponseProcessor.onMessage(createkafkaRecord("TestID1234", statusCode)) + verify(eventPublisher, times(1)).publishEvent(any()) + } + + companion object { + + @JvmStatic + fun statusCodeSource(): Set { + return setOf( + HttpStatus.OK, + HttpStatus.CREATED, + HttpStatus.BAD_REQUEST, + HttpStatus.NOT_FOUND, + HttpStatus.UNPROCESSABLE_ENTITY, + HttpStatus.INTERNAL_SERVER_ERROR + ) + .map { it.value() } + .toSet() + } + + } + +} \ No newline at end of file From 501762d4513ba6050e99e5e670a67a6cb672020d Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 9 Aug 2023 18:32:03 +0200 Subject: [PATCH 25/50] Add test logging --- build.gradle.kts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 78bad77..1f1a123 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,4 @@ +import org.gradle.api.tasks.testing.logging.TestLogEvent import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { @@ -76,6 +77,9 @@ tasks.withType { tasks.withType { useJUnitPlatform() + testLogging { + events(TestLogEvent.FAILED, TestLogEvent.PASSED, TestLogEvent.SKIPPED) + } } task("integrationTest") { From 2b42a4d262a846feb1f82facbb151be9cabb57b4 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 12:11:39 +0200 Subject: [PATCH 26/50] Tests for RestMtbFileSender --- .../processor/config/AppRestConfiguration.kt | 10 +- .../etl/processor/output/RestMtbFileSender.kt | 7 +- .../processor/output/RestMtbFileSenderTest.kt | 159 ++++++++++++++++++ 3 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt index 5e77a4f..a830597 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt @@ -28,6 +28,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.annotation.Order +import org.springframework.web.client.RestTemplate @Configuration @EnableConfigurationProperties( @@ -43,9 +44,14 @@ class AppRestConfiguration { private val logger = LoggerFactory.getLogger(AppRestConfiguration::class.java) @Bean - fun restMtbFileSender(restTargetProperties: RestTargetProperties): MtbFileSender { + fun restTemplate(): RestTemplate { + return RestTemplate() + } + + @Bean + fun restMtbFileSender(restTemplate: RestTemplate, restTargetProperties: RestTargetProperties): MtbFileSender { logger.info("Selected 'RestMtbFileSender'") - return RestMtbFileSender(restTargetProperties) + return RestMtbFileSender(restTemplate, restTargetProperties) } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt index 24cdc49..f80ff69 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -28,12 +28,13 @@ import org.springframework.http.MediaType import org.springframework.web.client.RestClientException import org.springframework.web.client.RestTemplate -class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) : MtbFileSender { +class RestMtbFileSender( + private val restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties +) : MtbFileSender { private val logger = LoggerFactory.getLogger(RestMtbFileSender::class.java) - private val restTemplate = RestTemplate() - override fun send(request: MtbFileSender.MtbFileRequest): MtbFileSender.Response { try { val headers = HttpHeaders() diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt new file mode 100644 index 0000000..17d420a --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt @@ -0,0 +1,159 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.output + +import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.config.RestTargetProperties +import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.springframework.http.HttpMethod +import org.springframework.http.HttpStatus +import org.springframework.test.web.client.MockRestServiceServer +import org.springframework.test.web.client.match.MockRestRequestMatchers.method +import org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo +import org.springframework.test.web.client.response.MockRestResponseCreators.withStatus +import org.springframework.web.client.RestTemplate + +class RestMtbFileSenderTest { + + private lateinit var mockRestServiceServer: MockRestServiceServer + + private lateinit var restMtbFileSender: RestMtbFileSender + + @BeforeEach + fun setup() { + val restTemplate = RestTemplate() + val restTargetProperties = RestTargetProperties("http://localhost:9000/mtbfile") + + this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate) + + this.restMtbFileSender = RestMtbFileSender(restTemplate, restTargetProperties) + } + + @ParameterizedTest + @MethodSource("deleteRequestWithResponseSource") + fun shouldReturnExpectedResponseForDelete(requestWithResponse: RequestWithResponse) { + this.mockRestServiceServer.expect { + method(HttpMethod.DELETE) + requestTo("/mtbfile") + }.andRespond { + withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it) + } + + val response = restMtbFileSender.send(MtbFileSender.DeleteRequest("TestID", "PID")) + assertThat(response.status).isEqualTo(requestWithResponse.requestStatus) + } + + @ParameterizedTest + @MethodSource("mtbFileRequestWithResponseSource") + fun shouldReturnExpectedResponseForMtbFilePost(requestWithResponse: RequestWithResponse) { + this.mockRestServiceServer.expect { + method(HttpMethod.POST) + requestTo("/mtbfile") + }.andRespond { + withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it) + } + + val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest("TestID", mtbFile)) + assertThat(response.status).isEqualTo(requestWithResponse.requestStatus) + } + + companion object { + data class RequestWithResponse(val httpStatus: HttpStatus, val body: String, val requestStatus: RequestStatus) + + private val warningBody = """ + { + "patient_id": "PID", + "issues": [ + { "severity": "warning", "message": "Something is not right" } + ] + } + """.trimIndent() + + private val errorBody = """ + { + "patient_id": "PID", + "issues": [ + { "severity": "error", "message": "Something is very bad" } + ] + } + """.trimIndent() + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("PID") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("PID") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("PID") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .build() + + /** + * Synthetic http responses with related request status + * Also see: https://ibmi-intra.cs.uni-tuebingen.de/display/ZPM/bwHC+REST+API + */ + @JvmStatic + fun mtbFileRequestWithResponseSource(): Set { + return setOf( + RequestWithResponse(HttpStatus.OK, "{}", RequestStatus.SUCCESS), + RequestWithResponse(HttpStatus.CREATED, warningBody, RequestStatus.WARNING), + RequestWithResponse(HttpStatus.BAD_REQUEST, "??", RequestStatus.ERROR), + RequestWithResponse(HttpStatus.UNPROCESSABLE_ENTITY, errorBody, RequestStatus.ERROR), + // Some more errors not mentioned in documentation + RequestWithResponse(HttpStatus.NOT_FOUND, "what????", RequestStatus.ERROR), + RequestWithResponse(HttpStatus.INTERNAL_SERVER_ERROR, "what????", RequestStatus.ERROR) + ) + } + + /** + * Synthetic http responses with related request status + * Also see: https://ibmi-intra.cs.uni-tuebingen.de/display/ZPM/bwHC+REST+API + */ + @JvmStatic + fun deleteRequestWithResponseSource(): Set { + return setOf( + RequestWithResponse(HttpStatus.OK, "", RequestStatus.SUCCESS), + // Some more errors not mentioned in documentation + RequestWithResponse(HttpStatus.NOT_FOUND, "what????", RequestStatus.ERROR), + RequestWithResponse(HttpStatus.INTERNAL_SERVER_ERROR, "what????", RequestStatus.ERROR) + ) + } + } + + +} \ No newline at end of file From 002b0618cf813d48bbff2d287e16f607a4c73d73 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 13:35:35 +0200 Subject: [PATCH 27/50] Add tests for KafkaMtbFileSender --- .../processor/output/KafkaMtbFileSender.kt | 4 +- .../output/KafkaMtbFileSenderTest.kt | 169 ++++++++++++++++++ 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 9448e29..e7f9769 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -50,7 +50,7 @@ class KafkaMtbFileSender( } } catch (e: Exception) { logger.error("An error occurred sending to kafka", e) - MtbFileSender.Response(RequestStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.ERROR) } } @@ -79,7 +79,7 @@ class KafkaMtbFileSender( } } catch (e: Exception) { logger.error("An error occurred sending to kafka", e) - MtbFileSender.Response(RequestStatus.UNKNOWN) + MtbFileSender.Response(RequestStatus.ERROR) } } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt new file mode 100644 index 0000000..14bdd5d --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt @@ -0,0 +1,169 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.output + +import com.fasterxml.jackson.databind.ObjectMapper +import de.ukw.ccc.bwhc.dto.* +import dev.dnpm.etl.processor.config.KafkaTargetProperties +import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.* +import org.springframework.kafka.core.KafkaTemplate +import org.springframework.kafka.support.SendResult +import java.util.concurrent.CompletableFuture.completedFuture +import java.util.concurrent.ExecutionException + +@ExtendWith(MockitoExtension::class) +class KafkaMtbFileSenderTest { + + private lateinit var kafkaTemplate: KafkaTemplate + + private lateinit var kafkaMtbFileSender: KafkaMtbFileSender + + private lateinit var objectMapper: ObjectMapper + + @BeforeEach + fun setup( + @Mock kafkaTemplate: KafkaTemplate + ) { + val kafkaTargetProperties = KafkaTargetProperties("testtopic") + this.objectMapper = ObjectMapper() + this.kafkaTemplate = kafkaTemplate + + this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, objectMapper) + } + + @ParameterizedTest + @MethodSource("requestWithResponseSource") + fun shouldSendMtbFileRequestAndReturnExpectedState(testData: TestData) { + doAnswer { + if (null != testData.exception) { + throw testData.exception + } + completedFuture(SendResult(null, null)) + }.whenever(kafkaTemplate).send(anyString(), anyString(), anyString()) + + val response = kafkaMtbFileSender.send(MtbFileSender.MtbFileRequest("TestID", mtbFile(Consent.Status.ACTIVE))) + assertThat(response.status).isEqualTo(testData.requestStatus) + } + + @ParameterizedTest + @MethodSource("requestWithResponseSource") + fun shouldSendDeleteRequestAndReturnExpectedState(testData: TestData) { + doAnswer { + if (null != testData.exception) { + throw testData.exception + } + completedFuture(SendResult(null, null)) + }.whenever(kafkaTemplate).send(anyString(), anyString(), anyString()) + + val response = kafkaMtbFileSender.send(MtbFileSender.DeleteRequest("TestID", "PID")) + assertThat(response.status).isEqualTo(testData.requestStatus) + } + + @Test + fun shouldSendMtbFileRequestWithCorrectKeyAndBody() { + doAnswer { + completedFuture(SendResult(null, null)) + }.whenever(kafkaTemplate).send(anyString(), anyString(), anyString()) + + kafkaMtbFileSender.send(MtbFileSender.MtbFileRequest("TestID", mtbFile(Consent.Status.ACTIVE))) + + val captor = argumentCaptor() + verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture()) + assertThat(captor.firstValue).isNotNull + assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"eid\": \"1\", \"requestId\": \"TestID\"}") + assertThat(captor.secondValue).isNotNull + assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(mtbFile(Consent.Status.ACTIVE))) + } + + @Test + fun shouldSendDeleteRequestWithCorrectKeyAndBody() { + doAnswer { + completedFuture(SendResult(null, null)) + }.whenever(kafkaTemplate).send(anyString(), anyString(), anyString()) + + kafkaMtbFileSender.send(MtbFileSender.DeleteRequest("TestID", "PID")) + + val captor = argumentCaptor() + verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture()) + assertThat(captor.firstValue).isNotNull + assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"requestId\": \"TestID\"}") + assertThat(captor.secondValue).isNotNull + assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(mtbFile(Consent.Status.REJECTED))) + } + + companion object { + fun mtbFile(consentStatus: Consent.Status): MtbFile { + return if (consentStatus == Consent.Status.ACTIVE) { + MtbFile.builder() + .withPatient( + Patient.builder() + .withId("PID") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(consentStatus) + .withPatient("PID") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("PID") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + } else { + MtbFile.builder() + .withConsent( + Consent.builder() + .withStatus(consentStatus) + .withPatient("PID") + .build() + ) + }.build() + } + + data class TestData(val requestStatus: RequestStatus, val exception: Throwable? = null) + + @JvmStatic + fun requestWithResponseSource(): Set { + return setOf( + TestData(RequestStatus.UNKNOWN), + TestData(RequestStatus.ERROR, InterruptedException("Test interrupted")), + TestData(RequestStatus.ERROR, ExecutionException(RuntimeException("Test execution aborted"))) + ) + } + } + +} \ No newline at end of file From b956eba6c746e511028e257ae102164867c70766 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 16:44:26 +0200 Subject: [PATCH 28/50] Add workflow to run tests on push or pull request --- .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ build.gradle.kts | 11 +++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5ff199b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: "Run Tests" + +on: + push: + branches: + - 'master' + pull-request: + branches: + - '*' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Execute tests + run: ./gradlew test + + - name: Execute integration tests + run: ./gradlew integrationTest \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 1f1a123..7650575 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,6 @@ import org.gradle.api.tasks.testing.logging.TestLogEvent import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.springframework.boot.gradle.tasks.bundling.BootBuildImage plugins { war @@ -90,3 +91,13 @@ task("integrationTest") { shouldRunAfter("test") } + +tasks.named("bootBuildImage") { + imageName.set("ghcr.io/ccc-mf/etl-processor") + + environment.set(environment.get() + mapOf( + "BP_OCI_SOURCE" to "https://github.com/CCC-MF/etl-processor", + "BP_OCI_LICENSES" to "AGPLv3", + "BP_OCI_DESCRIPTION" to "ETL Processor for bwHC MTB files" + )) +} From 1e29ecc891fdfa9750af49e5a3506b6fe5ad8f46 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 17:32:02 +0200 Subject: [PATCH 29/50] Fix event name in workflow file --- .github/workflows/test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ff199b..d7d3e39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,11 +2,9 @@ name: "Run Tests" on: push: - branches: - - 'master' - pull-request: - branches: - - '*' + branches: [ 'master' ] + pull_request: + branches: [ '*' ] jobs: docker: From d24d9a7fd0ddd725bb81da29e40b80a644c66249 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 17:46:11 +0200 Subject: [PATCH 30/50] Add docker deploy workflow --- .github/workflows/docker.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/test.yml | 5 +++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..b74eaac --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,30 @@ +name: "Run docker build and deploy" + +on: + push: + tags: + - '*' + +jobs: + call-test-workflow: + uses: ./.github/workflows/test.yml + + docker: + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Execute image build and push + run: | + ./gradlew bootBuildImage + docker tag ghcr.io/ccc-mf/etl-processor ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }} + docker push ghcr.io/ccc-mf/etl-processor + docker push ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7d3e39..98067f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,13 +1,14 @@ name: "Run Tests" on: + workflow_call: push: branches: [ 'master' ] pull_request: branches: [ '*' ] jobs: - docker: + tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -17,7 +18,7 @@ jobs: distribution: 'temurin' - name: Setup Gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.4.2 - name: Execute tests run: ./gradlew test From 2ec5e27a402f36d87d3cf9a6369e77028200c3b2 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 18:07:25 +0200 Subject: [PATCH 31/50] Full setup for docker build --- .github/workflows/docker.yml | 14 +++++++++++--- .github/workflows/test.yml | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b74eaac..175250c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,16 +2,24 @@ name: "Run docker build and deploy" on: push: - tags: - - '*' + tags: [ '*' ] jobs: - call-test-workflow: + test: uses: ./.github/workflows/test.yml docker: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.4.2 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98067f1..fa00f69 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,7 @@ on: workflow_call: push: branches: [ 'master' ] + tags-ignore: [ '*' ] pull_request: branches: [ '*' ] From 2264d85bd10cf18c8a6e7e0c7fb8f79bf7c24b87 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 18:29:25 +0200 Subject: [PATCH 32/50] Run docker workflow after test workflow --- .github/workflows/docker.yml | 9 ++++----- .github/workflows/test.yml | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 175250c..bf4f6a0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,14 +1,13 @@ name: "Run docker build and deploy" on: - push: - tags: [ '*' ] + workflow_run: + workflows: [ 'Run Tests' ] + types: [ 'completed' ] jobs: - test: - uses: ./.github/workflows/test.yml - docker: + if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.ref, '/tags/') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa00f69..e7ee0d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,8 @@ -name: "Run Tests" +name: 'Run Tests' on: - workflow_call: push: branches: [ 'master' ] - tags-ignore: [ '*' ] pull_request: branches: [ '*' ] @@ -24,5 +22,17 @@ jobs: - name: Execute tests run: ./gradlew test + integrationTests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.4.2 + - name: Execute integration tests run: ./gradlew integrationTest \ No newline at end of file From 25ec557c25378774194627775b460af290bfccee Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 18:44:33 +0200 Subject: [PATCH 33/50] Change condition when to run docker job --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bf4f6a0..d6f3e3e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ on: jobs: docker: - if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.ref, '/tags/') }} + if: ${{ github.event.workflow_run.conclusion == 'success' && github.ref_type == 'tag' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 55153d805048a91c10543969e583c12b23e10c78 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 18:55:33 +0200 Subject: [PATCH 34/50] Add information about docker image --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea0c02b..33148df 100644 --- a/README.md +++ b/README.md @@ -53,4 +53,8 @@ Wird keine Rückantwort über Apache Kafka empfangen und es gibt keine weitere M Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. Lässt sich keine Verbindung zu dem bwHC-Backend aufbauen, wird eine Rückantwort mit Status-Code `900` erwartet, welchen es -für HTTP nicht gibt. \ No newline at end of file +für HTTP nicht gibt. + +## Docker-Images + +Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor \ No newline at end of file From 4dde13e79a0d7d35648c498f7a98b62aea05b9ec Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 18:59:04 +0200 Subject: [PATCH 35/50] Run tests on each tag --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7ee0d3..f7c37f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ name: 'Run Tests' on: push: branches: [ 'master' ] + tags: [ '*' ] pull_request: branches: [ '*' ] From 044d01534b1183449bfe7d2a783481b81feac455 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 10 Aug 2023 20:17:10 +0200 Subject: [PATCH 36/50] Build and deploy docker image on new release --- .github/workflows/{docker.yml => deploy.yml} | 8 +++----- build.gradle.kts | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) rename .github/workflows/{docker.yml => deploy.yml} (81%) diff --git a/.github/workflows/docker.yml b/.github/workflows/deploy.yml similarity index 81% rename from .github/workflows/docker.yml rename to .github/workflows/deploy.yml index d6f3e3e..6d15376 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/deploy.yml @@ -1,13 +1,11 @@ -name: "Run docker build and deploy" +name: "Run build and deploy" on: - workflow_run: - workflows: [ 'Run Tests' ] - types: [ 'completed' ] + release: + types: [ 'published' ] jobs: docker: - if: ${{ github.event.workflow_run.conclusion == 'success' && github.ref_type == 'tag' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/build.gradle.kts b/build.gradle.kts index 7650575..ed8056d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,7 +11,7 @@ plugins { } group = "de.ukw.ccc" -version = "0.1.0-SNAPSHOT" +version = "0.1.0" java { sourceCompatibility = JavaVersion.VERSION_17 From cb9c5904729c90b86357d0668604b74f4f4e61f7 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 11 Aug 2023 09:13:45 +0200 Subject: [PATCH 37/50] Issue #2: Do not serialize JSON string as custom string (#4) In addition to that, if REST request did not contain a response body, use empty string as data quality report string. --- .../etl/processor/output/RestMtbFileSender.kt | 2 +- .../processor/services/ResponseProcessor.kt | 4 +- .../processor/output/RestMtbFileSenderTest.kt | 60 +++++++++++++++---- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt index f80ff69..1c59f5c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -50,7 +50,7 @@ class RestMtbFileSender( return MtbFileSender.Response(response.statusCode.asRequestStatus(), "Status-Code: ${response.statusCode.value()}") } logger.debug("Sent file via RestMtbFileSender") - return MtbFileSender.Response(response.statusCode.asRequestStatus()) + return MtbFileSender.Response(response.statusCode.asRequestStatus(), response.body.orEmpty()) } catch (e: IllegalArgumentException) { logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) } catch (e: RestClientException) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt index d7ad86f..f2e9e2e 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -55,14 +55,14 @@ class ResponseProcessor( RequestStatus.WARNING -> { it.report = Report( "Warnungen über mangelhafte Daten", - objectMapper.writeValueAsString(event.body) + event.body.orElse("") ) } RequestStatus.ERROR -> { it.report = Report( "Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar", - objectMapper.writeValueAsString(event.body) + event.body.orElse("") ) } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt index 17d420a..78b5a45 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt @@ -61,7 +61,8 @@ class RestMtbFileSenderTest { } val response = restMtbFileSender.send(MtbFileSender.DeleteRequest("TestID", "PID")) - assertThat(response.status).isEqualTo(requestWithResponse.requestStatus) + assertThat(response.status).isEqualTo(requestWithResponse.response.status) + assertThat(response.body).isEqualTo(requestWithResponse.response.body) } @ParameterizedTest @@ -75,11 +76,16 @@ class RestMtbFileSenderTest { } val response = restMtbFileSender.send(MtbFileSender.MtbFileRequest("TestID", mtbFile)) - assertThat(response.status).isEqualTo(requestWithResponse.requestStatus) + assertThat(response.status).isEqualTo(requestWithResponse.response.status) + assertThat(response.body).isEqualTo(requestWithResponse.response.body) } companion object { - data class RequestWithResponse(val httpStatus: HttpStatus, val body: String, val requestStatus: RequestStatus) + data class RequestWithResponse( + val httpStatus: HttpStatus, + val body: String, + val response: MtbFileSender.Response + ) private val warningBody = """ { @@ -123,6 +129,8 @@ class RestMtbFileSenderTest { ) .build() + private val errorResponseBody = "Sonstiger Fehler bei der Übertragung" + /** * Synthetic http responses with related request status * Also see: https://ibmi-intra.cs.uni-tuebingen.de/display/ZPM/bwHC+REST+API @@ -130,13 +138,33 @@ class RestMtbFileSenderTest { @JvmStatic fun mtbFileRequestWithResponseSource(): Set { return setOf( - RequestWithResponse(HttpStatus.OK, "{}", RequestStatus.SUCCESS), - RequestWithResponse(HttpStatus.CREATED, warningBody, RequestStatus.WARNING), - RequestWithResponse(HttpStatus.BAD_REQUEST, "??", RequestStatus.ERROR), - RequestWithResponse(HttpStatus.UNPROCESSABLE_ENTITY, errorBody, RequestStatus.ERROR), + RequestWithResponse(HttpStatus.OK, "{}", MtbFileSender.Response(RequestStatus.SUCCESS, "{}")), + RequestWithResponse( + HttpStatus.CREATED, + warningBody, + MtbFileSender.Response(RequestStatus.WARNING, warningBody) + ), + RequestWithResponse( + HttpStatus.BAD_REQUEST, + "??", + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ), + RequestWithResponse( + HttpStatus.UNPROCESSABLE_ENTITY, + errorBody, + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ), // Some more errors not mentioned in documentation - RequestWithResponse(HttpStatus.NOT_FOUND, "what????", RequestStatus.ERROR), - RequestWithResponse(HttpStatus.INTERNAL_SERVER_ERROR, "what????", RequestStatus.ERROR) + RequestWithResponse( + HttpStatus.NOT_FOUND, + "what????", + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ), + RequestWithResponse( + HttpStatus.INTERNAL_SERVER_ERROR, + "what????", + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ) ) } @@ -147,10 +175,18 @@ class RestMtbFileSenderTest { @JvmStatic fun deleteRequestWithResponseSource(): Set { return setOf( - RequestWithResponse(HttpStatus.OK, "", RequestStatus.SUCCESS), + RequestWithResponse(HttpStatus.OK, "", MtbFileSender.Response(RequestStatus.SUCCESS)), // Some more errors not mentioned in documentation - RequestWithResponse(HttpStatus.NOT_FOUND, "what????", RequestStatus.ERROR), - RequestWithResponse(HttpStatus.INTERNAL_SERVER_ERROR, "what????", RequestStatus.ERROR) + RequestWithResponse( + HttpStatus.NOT_FOUND, + "what????", + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ), + RequestWithResponse( + HttpStatus.INTERNAL_SERVER_ERROR, + "what????", + MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + ) ) } } From 6ecb439007b4fa6dec9af1e0334b89fd235a97be Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 11 Aug 2023 09:22:54 +0200 Subject: [PATCH 38/50] Issue #3: Detect the request type of request with last known status (#5) --- .../services/RequestServiceIntegrationTest.kt | 2 +- .../processor/services/RequestProcessor.kt | 2 +- .../etl/processor/services/RequestService.kt | 7 +- .../services/RequestProcessorTest.kt | 8 +-- .../processor/services/RequestServiceTest.kt | 64 ++++++++++++------- 5 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt index 3af218e..ff85296 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt @@ -116,7 +116,7 @@ class RequestServiceIntegrationTest : AbstractTestcontainerTest() { fun shouldReturnDeleteRequestAsLastRequest() { setupTestData() - val actual = requestService.isLastRequestDeletion("TEST_12345678901") + val actual = requestService.isLastRequestWithKnownStatusDeletion("TEST_12345678901") assertThat(actual).isTrue() } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index d2f8619..34156f7 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -95,7 +95,7 @@ class RequestProcessor( private fun isDuplication(pseudonymizedMtbFile: MtbFile): Boolean { val lastMtbFileRequestForPatient = requestService.lastMtbFileRequestForPatientPseudonym(pseudonymizedMtbFile.patient.id) - val isLastRequestDeletion = requestService.isLastRequestDeletion(pseudonymizedMtbFile.patient.id) + val isLastRequestDeletion = requestService.isLastRequestWithKnownStatusDeletion(pseudonymizedMtbFile.patient.id) return null != lastMtbFileRequestForPatient && !isLastRequestDeletion diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt index 0f69910..e0043d2 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt @@ -38,8 +38,8 @@ class RequestService( fun lastMtbFileRequestForPatientPseudonym(patientPseudonym: String) = Companion.lastMtbFileRequestForPatientPseudonym(allRequestsByPatientPseudonym(patientPseudonym)) - fun isLastRequestDeletion(patientPseudonym: String) = - Companion.isLastRequestDeletion(allRequestsByPatientPseudonym(patientPseudonym)) + fun isLastRequestWithKnownStatusDeletion(patientPseudonym: String) = + Companion.isLastRequestWithKnownStatusDeletion(allRequestsByPatientPseudonym(patientPseudonym)) companion object { @@ -48,7 +48,8 @@ class RequestService( .sortedByDescending { it.processedAt } .firstOrNull { it.status == RequestStatus.SUCCESS || it.status == RequestStatus.WARNING } - fun isLastRequestDeletion(allRequests: List) = allRequests + fun isLastRequestWithKnownStatusDeletion(allRequests: List) = allRequests + .filter { it.status != RequestStatus.UNKNOWN } .maxByOrNull { it.processedAt }?.type == RequestType.DELETE } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index f9d8182..7856833 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -92,7 +92,7 @@ class RequestProcessorTest { doAnswer { false - }.`when`(requestService).isLastRequestDeletion(anyString()) + }.`when`(requestService).isLastRequestWithKnownStatusDeletion(anyString()) doAnswer { it.arguments[0] as String @@ -147,7 +147,7 @@ class RequestProcessorTest { doAnswer { false - }.`when`(requestService).isLastRequestDeletion(anyString()) + }.`when`(requestService).isLastRequestWithKnownStatusDeletion(anyString()) doAnswer { it.arguments[0] as String @@ -202,7 +202,7 @@ class RequestProcessorTest { doAnswer { false - }.`when`(requestService).isLastRequestDeletion(anyString()) + }.`when`(requestService).isLastRequestWithKnownStatusDeletion(anyString()) doAnswer { MtbFileSender.Response(status = RequestStatus.SUCCESS) @@ -261,7 +261,7 @@ class RequestProcessorTest { doAnswer { false - }.`when`(requestService).isLastRequestDeletion(anyString()) + }.`when`(requestService).isLastRequestWithKnownStatusDeletion(anyString()) doAnswer { MtbFileSender.Response(status = RequestStatus.ERROR) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt index 3e0a979..3cf8804 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestServiceTest.kt @@ -68,23 +68,33 @@ class RequestServiceTest { patientId = "TEST_12345678901", pid = "P1", fingerprint = "0123456789abcdef1", - type = RequestType.DELETE, - status = RequestStatus.SUCCESS, - processedAt = Instant.parse("2023-08-08T02:00:00Z") - ), - Request( - id = 1L, - uuid = UUID.randomUUID().toString(), - patientId = "TEST_12345678902", - pid = "P2", - fingerprint = "0123456789abcdef2", type = RequestType.MTB_FILE, status = RequestStatus.WARNING, - processedAt = Instant.parse("2023-08-08T00:00:00Z") + processedAt = Instant.parse("2023-07-07T00:00:00Z") + ), + Request( + id = 2L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdefd", + type = RequestType.DELETE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-07-07T02:00:00Z") + ), + Request( + id = 3L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.MTB_FILE, + status = RequestStatus.UNKNOWN, + processedAt = Instant.parse("2023-08-11T00:00:00Z") ) ) - val actual = RequestService.isLastRequestDeletion(requests) + val actual = RequestService.isLastRequestWithKnownStatusDeletion(requests) assertThat(actual).isTrue() } @@ -98,23 +108,33 @@ class RequestServiceTest { patientId = "TEST_12345678901", pid = "P1", fingerprint = "0123456789abcdef1", - type = RequestType.DELETE, - status = RequestStatus.SUCCESS, + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, + processedAt = Instant.parse("2023-07-07T00:00:00Z") + ), + Request( + id = 2L, + uuid = UUID.randomUUID().toString(), + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", + type = RequestType.MTB_FILE, + status = RequestStatus.WARNING, processedAt = Instant.parse("2023-07-07T02:00:00Z") ), Request( - id = 1L, + id = 3L, uuid = UUID.randomUUID().toString(), - patientId = "TEST_12345678902", - pid = "P2", - fingerprint = "0123456789abcdef2", + patientId = "TEST_12345678901", + pid = "P1", + fingerprint = "0123456789abcdef1", type = RequestType.MTB_FILE, - status = RequestStatus.WARNING, - processedAt = Instant.parse("2023-08-08T00:00:00Z") + status = RequestStatus.UNKNOWN, + processedAt = Instant.parse("2023-08-11T00:00:00Z") ) ) - val actual = RequestService.isLastRequestDeletion(requests) + val actual = RequestService.isLastRequestWithKnownStatusDeletion(requests) assertThat(actual).isFalse() } @@ -197,7 +217,7 @@ class RequestServiceTest { @Test fun isLastRequestDeletionShouldRequestAllRequestsForPatientPseudonym() { - requestService.isLastRequestDeletion("TEST_12345678901") + requestService.isLastRequestWithKnownStatusDeletion("TEST_12345678901") verify(requestRepository, times(1)).findAllByPatientIdOrderByProcessedAtDesc(anyString()) } From 0e1034d964639fe295726c2f3c8bc801a1ff7017 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 11 Aug 2023 09:47:20 +0200 Subject: [PATCH 39/50] New version and add status badge to README.md --- README.md | 2 +- build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33148df..43ed3c4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ETL-Processor for bwHC data +# ETL-Processor for bwHC data [![Run Tests](https://github.com/CCC-MF/etl-processor/actions/workflows/test.yml/badge.svg)](https://github.com/CCC-MF/etl-processor/actions/workflows/test.yml) Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisiert die Patienten-ID. diff --git a/build.gradle.kts b/build.gradle.kts index ed8056d..37fe4e1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,7 +11,7 @@ plugins { } group = "de.ukw.ccc" -version = "0.1.0" +version = "0.1.1" java { sourceCompatibility = JavaVersion.VERSION_17 From bc48a7217eb98e9ec95e5c8b0908b2a1d8a6b27c Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 11 Aug 2023 14:37:48 +0200 Subject: [PATCH 40/50] Add more information about usage in an ETl process --- README.md | 22 ++++++++++++++++++++++ docs/etl.png | Bin 0 -> 76404 bytes 2 files changed, 22 insertions(+) create mode 100644 docs/etl.png diff --git a/README.md b/README.md index 43ed3c4..a547ab5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,28 @@ Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisiert die Patienten-ID. +### Einordnung innerhalb einer DNPM-ETL-Strecke + +Diese Anwendung erlaubt das Entgegennehmen HTTP/REST-Anfragen aus dem Onkostar-Plugin **onkostar-pligin-dnpmexport**. + +Der Inhalt einer Anfrage, wenn ein bwHC-MTBFile, wird pseudonymisiert und auf Duplikate geprüft. +Duplikate werden verworfen, Änderungen werden weitergeleitet. + +Löschanfragen werden immer als Löschanfrage an das bwHC-backend weitergeleitet. + +![Modell DNPM-ETL-Strecke](docs/etl.png) + +#### HTTP/REST-Konfiguration + +Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung direkt an das bwHC-Backend gesendet. + +#### Konfiguration für Apache Kafka + +Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung an Apache Kafka übergeben. +Eine Antwort wird dabei ebenfalls mithilfe von Apache Kafka übermittelt und nach der Entgegennahme verarbeitet. + +Siehe hierzu auch: https://github.com/CCC-MF/kafka-to-bwhc + ## Pseudonymisierung der Patienten-ID Wenn eine URI zu einer gPAS-Instanz angegeben ist, wird diese verwendet. diff --git a/docs/etl.png b/docs/etl.png new file mode 100644 index 0000000000000000000000000000000000000000..0ca5def8756b51ad0beb36cff05bba5a1e49f999 GIT binary patch literal 76404 zcmd@6g;&+l+64-4x*O>hl6zT5n1}RBJI;BIprTZ?RM9uhx2U&iN3oASZ!_OoR-9K+rx(iYq}Nu;8Ciiiq&w+y3>cEBJ}GlI9m6tPi`*an>VAZs1opWtIc{`09qxcI7QB){MPq+3#MGjlgz z4vzZT(Mmqx&Wtb&y_z}1>CF3|6Ea1i)W8jni~Rquuyc02%KFQQZmHr2Di#Vj(vb7- zFMjd2*5)o>Y0EQp8wVgI?Eh9klBz+t+rv`aiBi286YqN`VqdLY%ywPZ8%>PBGw z;>LPl3}QM6uwHH;tcjyp+JSFth2I5=;y)J$FYpj4$p7!!Fc2EZ|2=3>9F!k66DAsDw|hCRaFybYk&Uy87E0y`J~Ue_F_Om?yruf?yjBP+y>`sjcUDa z9sBzFobOLQAW|ByZ4mPRXKx9b_xCOBc}W=cI!Pc3s;a@m$t;_z&rdv8hkpXg)vHAr z@n9g|q-cKq`h|&w&z;3ct5PX$I*;8)4Gn@fvS|f>1f6%s@vd)gcd`{VG{SDrcVHmK?1bR- z6bV1Iyfe49w${#XArLb&vr12ORaHuQdgRw1qq=X-Hn|of1l>g(!GhgrJx z!DAz$q1j*+YcorH_y9-db7OFGe=o+I{F;dg?Vamk_tnwj*+F&J#(F5ZEtlMjFLV9Z zKqZ_1{Q0x~c)dEea24zO0CUoDP1a<)9N}^>YjKp$??S)wWhYau+U?)pPhi#7F6Z zfBhO}XDoZJERhF3H#awh%?Q4(u1-NgL4VgCcqakne_g&nVra_|;+LALSv-b@j0_Dq z{8NuWpD0!BV6)hSMuLtIiN`c~;Nl6k5DtO}3k@Ry7Ojs!5&u8dvYRPY_~UK17c?Y= z$4)o$@zbYI^Ig7Ov)u8LL8%;O(B?7_#?MWti%mAYVOXzE7p$t+CnU&cE_zPrlT2!@ z=9p!&1xe-PFz*37B zMXl#+psK(BLr!ARLywo7F65QZ5(tlqLbYCK__X7r^LU+pesOU)>D|6DTlu-Bu1?g! zf$i(puQfJ{+}{!t4S@f2;K5J^#XSZ8pIbMDgusvBy1>D~4ULTro^1}Ynf$`d9IV!9 z$E)tRN}FxEw_4#NLD&7V_3HcS50v>TNU_b9C|8mVZkT4($&~cq>1!(&34Pg*8#`0HVvD(0>DM zoGrNPK42aYNO*WSu>XF1J+BE%XkkXY-v0hVu{peOarfiaof^Y_WUtd6g!vi+m=*79 z9k08+;>?KJrY1b_V7+-CBDHKp^zH00SXfvx@3jty#+5rC{z?B%<3cGaDgrTT?QGYI z7(D%Pt_TDPg#J8rC@3h64i7GLGJ&w&UGol|WCMZ2{qxJgZ()8yWRxV}^(GwUTmj1E1|W3jd6< zqG}`sE710VJE=9F$dl`@^S*bjVT=q$!FPRxdpYOFnExdLdBIy)#^I)UaFEZo-ME#XJ*2T#wFg^WqXcfpS zLC)~e|L-gY3J_`;BOmXt3HVV(yE&X8;zINc3|RPXI89C1uyAmM{3N1xq^;U9DJfQK z_PmHF#OQ+AcDr$id(Cn?YGNZq{1GuojU2(zNZy%LkDv8C;puT5trsrdFl+OgjPthT zwY6bOCs(mM6BtAC-zLQyY2vNg-jbJZ;AWd=X;brTDoTV0Qk+1c&% z&F4GJaxyc~HgHCJe@+nk`*$qvHxv~S*>$$!Sy=pZKdc>GJ2_b`zdm6sRf(wXyd&38 zLyMP$`1^Gxvbz;C|^%t;PrUBN&RK{;^DUF-Vv zUVcX!gqE4PF9gk3?8_IVEDuZY`0qWAR&+pY_OI@GB0>&^e#zG5R>ER|x%pH<*pQF1 zg-ZU5zw{KSvGcyGEVM&mO=39kR=SCs61?6kkO_;wxxXbDVdtSE{pFDgm2W}Z+=5E) z`57W?8%<)F1Q%+6O7e>GuHR;paA>dJEJsBv5MNgVwzL7Bo12^0_0YX~xt&3U{uRh_ zJ*kU^Ad#YF$q~& zG)qg%muJpZD+ga}?lr_RvZa57_CA9P{?*p;QczZgshyXx@)Y&rp6$A-@Xw7P$2{`@hP zD0X-*rh8brok3g55xq!uufw*;#DoUor&^x1*XZdgVQoz^n$>}bgOkP2o zqDxAu<5J_{dE7iz)ITYzX~hPkBsGI>D_QRepFR=Qs}thYi3wf>$jGy@bO|`#V3VMu znTjJqf-uPazka3a8{i%VtI^806-L0-I`Z@!it!wt&QmSS*aV4v+j38aBTN;qe>i`J z8QI@)iH*pHd?06HnWU;ZFcB=9m<5B{JY6Aw&xJL(jX=OkC3?Xi;6*JZ6^TXc3B_DB z3-MdZKo*Zmn>KxZ7J7bwad4QUz>7TbgkLZ?oOifDrS9{4Sxodttmw1_WL8x2a3syO zvl~!geDr#V<}QZ9jqE27XxSPnEi7!E@9=yJa@0bC0gfT3?Ml=KrAelgl(4uYEr_3g zV&bdGDP>GPQxX{&8Rx9Q%`;GFqt?!ATY1{pywA()9FS(|SV5Vv9OR&G$f63zx`SN_iiZxkE@GTe z!t)ryKjnz)#St|>Sg^Xbk{3g2%%K_Z$A&yAI5-g6t_*dj3{j+{hOccO z``z8ae}C@js#yy!W;4f}k-?nQ@g->sGB6Odcl_G@;|FT4K~dPk9pT5=tcbJCH$?mb zqEVuve>&pNTl)(uXP<_fgkYdz0}%uT1B7h&27VTGO+>3~VDm zO+j@&tgz7KUmW?@uQ^`L8QB_Yq>zhK;h+*92YVxQ4bD$+#ARiLl$9A~?wFdqS|5t< zMMNiq#>dCc9K>WF4@N^7q+rn_dH3`#1nt^ilDaQWfo17aTUbDV_Vf(Bf_4U6e90E% zQ)#Ituj60VjEoE{Y|--{w7Hq z{Ma%0-t}65dRyT{8-4UEZRTM8+0X78&*guDZAr0nKQdj&q{C{7i@v}=h1A&VrKBLT z8A}?KPX#<^PtX1WS6=?b!HOdhLjp=Hln_)xL%&@p^j<$Wm?YJ};q~sNXmU1Mff0oZjC*AlkYQ=^%1PSL#>2D+*{iLo#5bL zxTU2G=d6>hE%=c5jp@<^&}cf?g~){|rm`Ba4@41$X8SCJFJ)NY+_m=1jF6o*f7LhI zLKYIT%iL*gZ};mh?GHvNrh^iOuL#hBOVh%4VkoPLOuhRXoF2o!VjJ%TNKLir!r%yh_JN2wwjiLqyl%V_YWUv;E z`Z(nVC72z1dde0S$c|sgC_um!Vc3)#P!zQznK}kuw z_48W-K`+jY4RKzN8%_;1bevq8vR`QTsDIB|6zGYkBHoBX{Y9pkw&dbPMer9~(YP3g z!8~=w4`lrPn^k}ovo{dh;c$b7M9@>WKGumxF{daAsxM3tI!DMbC$aQlJ3W>Hd%67c zC)o~wz?53tvnyGkqmwj#e{FzmIGxrp=z2IGRaPc5(x##~u|9mb@G7`@^+_^2 z9?KAS-(@(n`{qwM*?_P zHMJCeQ_hEu@A_1@!i7qd?KPOmc1u4t>x-ybK1uX;u$tHtlnBzET{ zRqgd^>LH{Y=Hoh(@?7^>f0DEO1(@-5tC%&Ob_Gt;M!fpB9M`utpsVF#Y-n&1S-GuK zOufUhS&E`#y0==~9~>`IRyN)g@F?z8kz7L5pDp?PTo9*Db(}fm35<(0Mx4X`W;3q^bKc)e7&(?dq)xADw>c6T1 zCYsukbU%(!j>gU^>*^;MJ+%;fSl{E3QNkxCq?dhxfAe~YN9RHJ+lTlB)tvX54V z6lWq|_Bq{%0A55e@N_WMY3&zLj*_C)XG4MitDtR{B*hgS;1B^+}qm>w_Ap;FlIRfV?#^xCDx0D zVlVDGkoM4}AOvjE(o^mE_s#De*?Se_F+|&b`PRS+i=_`D!g}B5H&Cpz!v=Vu5|afc zyro7=>mY%i>SWW$p@gt5{3O9gDbR1#W+j2ISU^mhS?X`B8!vm5*ZkP{T=0BKZXJ1* zduGtr~(R-OFQoL|_``R=M&Eia+V7n50sAK5QhYUZXfKR5TRH+68+LUws@ z>tFZIRCa3IC@6vz%7bZ{tERpW{&ggrw!{NCAhf@*6Juzv-{2-O{WSW|Hb=}XvT#;> zX#<8*QP=v8cs(>*KgKLb`0OH%PWbaDoq z0yS=78}Uz8Rjs9JN6(iZ#8B~2DDS9nk!xKJIH@t%zkXb6z(`(a-8Sx zXm3=4PlKiKRv0COmJ+W65}tBvO;>rd{89bNn>XU)(q_xAxrzxPVexsBl)xU9E3~2y ztt8g=zEtyZ_12tT$cW$FcECqP#U;6uz`GVj%ZATT;lM~t3C1Y2mp)H?fvP0=gt+&K z5mG1>ELJ=wT58D`DJf4^UUAsyiIpe+g1*K1l+n_`lS~U!(akIz?=cyasK69{%j+uuMV%IFje>9r8$m@N_LYwX#_tTgi@}|Z69hsibmula-!orc| zr_UEqGdeSluVPG93HhVYJ*Vde#wiIZ0dev0aE=l8^Elc|TU#6P-8-^~2;{~UBhg$; z_S7XzgC>Uae|D#*GPY+|J1DsJ4kHo+g{!Q8b9c~iw1v9sx?#|X4VXw^(R)AmXz{C% zpyy?uLx$_tSW*9_bA2-D5iL>*e2ptys&W-xRPP?I*J#GFz<5W0bDI_x^P)Hy1~}h)1hk2uc=3v)t-) z18E2h41CDQhFptwt5yCo12E&Q81E*(jx`a}mMRl~RZ;H36|u->UT8isD=5yM`*pbZ zhs#G>ikhVIngH_m?{v>drQIc*bxEoCCoO@H(V^+qRnKwKjQ8jG&_zOUSv7Y3wgA=q zAgfDA!X)v7aT{KP8Q~J@$ zR0Itawpgfe*vjGJwQ{6cPKWmtN1VkO>hF=tz;dfq_ZZF*E&I3cVUXxb6;?Az{R{|7 zW-Lh_4yehp1ZsO9udVj?F}*i%v8N`_{!K|!sMFeSsR@kga{M-%PGb688!RR!P6iW%u5PY7ay;^)sOwHizN=Q3{+C|OuA0D=nI?11+6cjBZnoyMl7 zhDQsHz_vJXk>Ot073Z(XX)%&dNTP}EaX_HJ8zb3-L@x~oKUXL^b|%ECAzK1gCSp{lqwIX}NDlz0y&HJUF`;L7SA6qP7nD*_KkR zN+qwN4*Sc8e#u`$ec%&d`AeoPt(QpPavc1`#bzFOI5~6j^76!d(~l96ko+qvnG6jL zosU=BTo`oS*WpNnd`OgwWF{8WwX`CdS3FSOxgCqE&U7Tz1Db-vZ1e?PM$J{2-|v{Q zRnQQ0uvE-K1x4;e7 z%U8Mnx3)a1KDEuJd%K8<2^1uZsfx>&cizM;0_ZoQg6?nL ztf72Zi~QZqgB8e92PDI8WhAn$I2&AOjAPU^^ec%T^XCghBZa1fnZ4NKVDt7^^@QIK zOf$(M1+t}Yp~ zc8gzITN^jZGbAoCF}SZ!{Ds0gn5{f099qO9B8qrn64N+;j_2;e0oH!cZ~E7OmP(8$ zlneROntGv$)n#g?_Zs?>d*7-bR{XINopDR&wG-}Jy+F}72%){g6owSI^ah5L5dR!> zXc4r&AN+_+#+dUzDRF;MaK`tjOuI%$MSdpJ);`ZsIL%e~N9@aRqd@PTX7T}s{1*|~ zexm8nP;;QRK|sx|6}Zhkr;^KAw<2jq9E2={1 z=|8%G3n+S*`+B%t`iJXR-VUcCiBiGESsuu#si`+x$$G4A$J&*Tk@zgf<1Tge^-wuu z4N2`T2Y8cHQ)r|ix)Y3$pa!Fh*t0YfZoxkQGeTAOsNQ}aLru#Yn0L4pMfG&%I)sPo zm8tmgZBK&DX;x4sjU;-)D8|tJh7T2~6=}7H&i?v$NWL;IRk?UwT;i;kCtTN`6V>AgNo{h50B3ap=lI2_{;{s^tF4}NrJ507Vh~YL zg8+m7Mm{SFP<$3wLRa%f{~=J%$-ex>3JMKvrXBaW`eTNOfgu5~=wu^)uN&4Ewlk4I zy@~e04R*NhW+Rg2bb#0n0$K67w*iN2=7y3%L$o_QRB<-rUrVrBPSeiM&#wVid|G>R zcOpN9&*=?-FxK|>Blw)Q*Dm&^n2s-(oW|B)ln7E%$au`!hyWRJcXwa&MinF-~)6FFSo0tgxzGb z7^0Z}v1fD?osyFB=6>}#CM_)ukL`G&5hed)bO4AZ&@eEy7E^ERFI8?unIMQ3gM|SD zd&UHb$Q5lTvrjyVG$!Va&>F6sXp2P{?E2yb?e5SxQQ8l0l;lqO_ml;q2)>}ZUM)XE zPhc?)uWd)i)6x+~Ih3yi&W{$VA1@}GBR}g?-uhtN+``dnwIf@`Bf)C&m84>Zry<2( z$LyR@T-yf*^!AV+%u*g)uc8sW2fHL+p@pO>HXM+UTVoS7qN<<}6M;;3Kw0z?=rC}2eq z(L)1)G6T(Ou4+d6WRTof+{C1Cd%g;A_W6`FWBTdKa*oPB`^#VM zw7en&&l~SB(=fh&8P!bs@af|lV^kuP=YASmtqyF)Ug0=Bpe0+Z5PChlCAz)=`zki{ zTRKl8@-ugQJpVC}BNZ@wyHrx4(X)6KhGC$kl`t^iW9LALPyZaCm=BFbK(J70rKPF4 z{`Y6XnmrM*Q#DvLARN$WcV&7Jk-Ug%R=v*AuE?%pGq+3|{INl6S7CpPOy_ zVK4-8RL$-FjlMfs8;XCO!t0P{`Wc80W~+1}ffH{mwK}Es+^u+>%SI9NptE$|P?5Q> zLCw!ErBl5FyG-u;!1kgo7->Snd7}^Nmav$Zf%59k2tZl`c*fvlrNjN-&$pkWpk6{n zPDKT3v+eREHG*2TZuHuk{`ZG}!|s3MB^uhS`uh7R-@n(v83genO0(XSlAoW5JrxyH z&LBOzpO5gY@9cyEJGU}5yx9?Y4qwxMl!=xGa8%oYW6e|ov=on z>0`9hA9vr1cBt1>cN-)E@Y~pC+6zyVH~FI--d?Q0lzX8c(MbpNSwzr2|-D5+P}t0(Z)Wm*UBjS69th4P|RYMcAjU3s?; zD6?2tr`nBWZhAlqbD@ld^%|3?Ec-{CV0izw*1_$q2`yczsHrKXqoWk7V}TGOQ-4jf z+u^J?@Qot62N=KTXtbBIFIMml@kO}=yr_Hg%zmQ}Nfw#{EJ1vHJOSX=Up7ERP3_08 zU&z3*LV)Haig1Am``^ERr$D-(v-9XOBc7a`EXIg;P?_psg~y~-0BS=bz0W{zFk7LS z&!r@kJvR8VF@t2Ds0ipJ-5g^A`F{pkQGpVr$|W-#=WT#N3i{=(ow<4U)8qXa(5qM? zKSL;Rkf^-X)yDh8&JgL{NZ`Yjnv{(?=i>)qbe#GfitfIB|u4a^`O((`(iG% zd`apb8~u@ISBrMd#GgF@voFbzCFFB5Vkgu1EWC-pAyFE1cO1HgpngsVH6E+M>tVRD z&*ES*x5(=@i=0m-Cqg^D>KyRu)ohEi0kn4LsPL&6!~I`IXaO?%ZNZYp z>YESCvLu>?*uI^2^!j!fPC`NuznL(sQ>_jnuCucm#3S8G3?LKfo}DG~YXk+Gy3u+U zVrwf+3M(ZnH#c5M^~uv(fUd4CQ`7CT_jW;XvC+-xIsum@u3UzBMrBb;3jschZXam; zLp~+EF?hVYGIMN2{E^|`ExJ~IJ-)u&8JV%OjIs2aXgKZ=KMwBm6er6o+3E$lCU6L zhrll@)AA%|W=1b8Xl1pgTuzI9IR!mE3<%J^nO`>UTIq|w zgtmy0<_NA~OU<$0%8Lo}p8?u0JaT6?hjOZvlJMdM)U5U0DO)F-+fOVG`$0$_qxsPo z(vLCjG~!NzQp)qOIPW#xKm48*D;S7WERVnNeTE(Tg$&6-M1irgLF@4jyywJP^j-MeY4EEPGf;9jX2m=*+E;PVgZhwI^P;|&sJfskUU%6!*zTyF1f98lod^P)yX z(8ndK2mDM3rKIR44Gw+;sfVjBdtcR{2 zAEN^$&;u|djT(I@xeQ)^pkXwh;dToK-PCBdQ0(;dv@1&4%N_x>6xV^Pt1GD-gtKFG zd)jSKkClzh;__xe>u9fbWMt$s$s(bPOEd=)94u^u`TL2#PvWx523<^2CPtA)%F0uC z_l%5;{KQ3aFEoCrug>oYMyG_y%TvYWdyPh-C@r)rTb$Ztc{2cfXRq6TA-LM*H`cux zY!zxjB=vY@d(3x~Js;7aqDQ@g{^8~a&)Tk#c5FOR^}lT2&gMEtnzaKIulveTqe@8F zJz4KPPko=SeQn6@kK0c+mfcp5uHL8fzIz8_UXf^Li8MjYz#x4< z;O*dSeK-KtKIZaZ?%=2mNU4O44*c);?%Lg6D*TD{Bk3<`CW$K7+WS4Wvn`Dlf)GDV zlETK=h|$|iAh6JCVs2^xUO@0i_FXK@&u)*Xd4~WqAzo^C{g45Z`wBw z26c6T2>SD9&c8doef@A1zX+znf-om}K+Tw6P;hE{Zf*1l0?C&^=pK}KuHPZybA$yc zV{XCqbWH?E2T{paQo7063}GuOD;zkSpn)cT)U0aW z867XRz6jL>A5O(yWN{q)zIH$qho|d(sd~8Fj;~xSH?wdJvMdBrB$GN>XvNdmWL|b= zx3Rk$78i#pE+LT*(p^(ilTO>yBNY`DgcE(?=4Lb2>gb)%LyPXQ2*_Q>=PpYfd&l|fooQcos@^&OCCxej|w51k@dd_r>1bQU05Qz94AH{FwfTe$0hh0fXkd`!E z$JI-s*i5`bXkRmihpS)C?p60#nuU{i*J=#kqVvD4>ge5=(F4kZ&U014{`P@67Ea$l z6OGkRc-rTjbCTQ1*H;4;L5wCbPygA+{qe{y1pVi(s=B(E3s1O`vU0u!6%qs}Wg|>0 z87!@=xMcM&dw%>7wNhY+mn@k1{cUW=D(&lvFC(S65CC&w3YEh5rUr@mzoCDUlap|8 zK>FU{S+unkEA$iw$_T@|i#>f!JUy>7xEH^D#h_*mpStPT3Ph9aK6iU>L29$yu1D|# z`o&(5;#R${Nl8dZE|fIzBm3v;Oi(~!8|tt*V6fcg^3tY)pDBR8e|lQw)LlT}_HV*+ z=iK8Uiu?*ak_ zeM`%TXkwlhRqIQ4vf}&fjY`N4%f-dj;(gCurPmeX`*@(2*{Nr5&srpx8IkRChXhcc zevsWL6MotQt9m#aB-imkyMB=hDbxN1vStoTtIv2(9)VhweH>#I{=pxfQ^^j4yVyC0 z%YGe?v9!DHd>C<5hF^NdmreD|YcuSxJ@Cs`=q^;n$O+4LW2@yn4^Q{ilJfH0LjdcbG5zfh z3k4;%AW+p*ff*43uc4*tvL~orh0Oic<6FUU!2Rh@oi=!jzxM;gC9?7o`$e*tJn85^ zZ#!jte3gLc2;`jU*;!Hg#JsvXrFvZu84Wv1g;yt52L?bA^AaQnN=!`TU=UmZT^%4y z0d%{B?gC_ehHL;>HUI^Dtp+Lz3Q9_9YR~WAs7Xml0CAbC_^he<@?W6mTRv`bgucS7#UIJg&sMZ>{ijO4hH<+ zvVdMUEDZ6b?E!ii9*|$=^0s4ziISU zzz7?&$S6`UTybPgd9#0?_DlAb2J287>phXWZ=Kl~HxP2q0UU8M&9T_6L{U>S`g3!Y z(ZTGa4Vh+AA|ml8o#rCgh6Lk0>Pz7AX|=kI$QkVzUZv#>--?QAEKRhid)9h@lJl~) zRnq&}yW}`#7X^ZjA&7XN%Hc_~T?JAuEghZ6hY!KC@}N~~U$nGr%cecDE0`Q8@B;6y zRF7<)=>vrb0|SG`=g-X=ww;w0I#lu9D=VFgfnHdXia_k{e$<#)QdYJ*-febdVP0bQ zoFgblUn!xf$>rq*y&%-`3N$1AV31%P?Cs&9HQ+X}K$8ieh-tpl6(G@N0~?ze-V0QC z$S0lmwviTlkTphDEVLM4H)l8fO^~^m4uqkGK#@A%;_!N7V22_}-v(29K3s?x^Uq>)I5q3PArvk!% zrJB`SaA;{dw}^|Lo(M1yMF35ckh$lmi>lM&zL<%H1zkjBB=FRz$w{SeNlAGH1+X7J zd`P=2BG)c_#o{RA4G2$1J_8=^?%ln;`6EWoSmEOQC#wrr_k8c(HNRWa`+Wbo z9SFwV%J1C;bklWEVWCCJcetO^S6DbXadWR_$(Q1N1FsaqyAkOUk-y&`!)v!XlAUjl zO!3JZ%UkN+T%S4kH@ls@^6^}-S|4?PwBGddu#O|Qf1fM1)5wjeFCXXo95s7anSp%B zn>@MAH-hq9jar-v;onHsbETr+M-dkn-!-w%7&T{n@Qn$9Cj{daY|iEff#TeB!%<>2 zzM2OJHNqnPIERPQ-J5%(9-lv}3)5K%%YGv3bG;*P7}~921fvEBTBUG`Lf0hvR*`A4 zV5SXE{ykje+bqAjKPqBsZdOQA@>$}?RXWRZkRfAsTs%A~KwG?Q=W1(fE0Cvt1m$Ez zM1%q;O&Q=kYBee>vTQ-1x3aPtscUWh5?OR0dmx%yGXw2&9^m7fqd-fMD8;2i1gIN; zHJNs_wY^B{OUlc|073y&d`0a+RaH2^ZA&UDrird|^1-kT=wsuOljoj#%ozp1es}lx z!vSZqcX0^@l`de?*4EZu=5K&Y{pjiGfiM+F>ZxdJht28vk~KEYxbK%1deA)(w?{+} z{r&sx$MP~U2+>@9hL18^A0^clp}U7@6QwFXUo%clC9t|4@`8{6u!fGgV?ABn7?95` zke@{n~{jge*w5 zdTV_sd_JR2{+j>J42<~n&duQy5)vL|7rdoU#C{oc!hK?5TD0S*t1kC%_>Rrpw8eid zUAW!i;LTZ9R?)Nd3rpZHk@m9VXOhA75(6vsz^~K1Qjn#>PP-L-CS||*V1P|YVuU1!4X)r&U zP((vR8~XKYQYL?fh&dU>)6=u1y}fs4<`sxCdmTCUE{1=J!!>R|El^|VnFr9jnJ zoDXV(MV%l}Qh|~=@ZrI$M8&eNZ)se);eB~UMeV)3oE#E8y`{dDm3*-8q}h1F;!r^$ z4dvUnFqT|aHQNdTWC)#|kM*Y*kS?JOjO}{qXa+HO7#?gi;&JaM&jO=L?6i^(*B{Gu zNrY2xt|SY&THMcYA#0L8VnjAhuUz*P*2y8i>ah(BDq*v$bzTqEp-xVD+3&cr zE``eX?xqeae04=}KdYa^{dRcBHfVdT}=;9CYbo%!$0=k`*h#3 zc>ObM78|Z~x_n3xFvzBH|D_|}(9qlwvPKMU_N=Us#l*xgrG0q^;!@?};v%szD&eHJ zEE7}OHSrN*qEulbDu|I2`&3GxLHc&mG%96U~lj zWP$Pdu?(cS;;ewODs;YguI6bQPO#DSUk;aA>!p=%K_fQ5u<+y0A3`v9Rnyco%gFP5 zbbKtwbzYzfM1|<+=%i%gQg0G+LHqRIxu&^?s*;lNg`~{P<^3EjVI%dhD=i#Ow5-B| z&?miX@3XVonudn%Af$MYR$W(DGD9Of!~JOsf_G{{bBD4Z_fI|eVj%7=AGZZH1PgwO z{#IO!aN|X(QZ`H8R2l!12xwb@Bw|76Mvw#@UwG9LdliS71rdHnD7q4oLLpSs{8ftt zCtGnEiDPWu!I&X1Om68ZlbagwmVImwL2kCQ+Pj^ ztg$!HeZn^f1u${l(QL{1Y8pzQHE7C^+4Z-U9{o+>spbl$DP^SszSPqt69|$ER(CG! z^MSIQnVH$%$!Ur+)3%Jx#mz0WzP?^JJr-wD5s(BNC2HsK)hERYv~k(lv0z?JRy_&u zq2RU1fqvno>Q6}_0Kj^~Jdks*ZEwTd+1cIP+;oG8=HugYdVU_5oUESQIfeO%g9v+i z2{b8zs5+@n)m3uHV9M_6S0#YmzNVmn#K#+5keg$G_Ldh!BnoUK1w9tKXIN1WEFdrb z799LOAn-G}O&ycxwNGeUn^@)FQ=kFFP8z2`olBUBb#xY8+m5Gl;o0cCUGV6) zu(2imM~(S!%$dZ*o>+siCxbyqIG4oV&@Bt00cTx<#3Z4Z3u8O*v3q(HA7}16s?(=M zCW7DAnXGrI(X=w(BU)gxUo*5L=H@0-)G|nXvRv|$eaUeJ1qBp*f@D|O-tbj17M_rt zh~eP0UjNG&QV67~kcI#qVase(IPGiRCkU$m8GOeD_HP(sA*Vq=UX&^q{}>rT1M$$_ z&8_%pqHp)?_Q6%+(bn92TnZh~`=B}%J5NYXE>2Zl_mL`8a$3^!eFW$T2pzq>VhnU^ z2L~v6dU^n{822ns=Ho>TNKl>|AYN$LQ+)wacPErV;7)nxRyEQ1vEAt{5}2S2)nJ59 z9`=_h3R0NAa~=u0;08@bXJAuMyGNyADU~c33*{eM24(>?I+vs5!t6W8?ZUkvOix7xmW5&w>s_o7#pj0(_^Ic_5*eJ!2;X#s$~P?QpZ;2@OEMMbzSE|a=p zCC@|dt^|zFa~kQxm*+wBd{Z@0E^)t}bc_DXXhUM+xv~$eevdF2n9e@o$A^Sysi^Lm zQD4Vcme38Rm_5OHuI75QX3ej>iXZb1T1wwhP}hZZdYy(|Q&xrys3cKsZ4%Jcal$>0 z%4P2S{R;~Q1!3JiVXmXsRuyGs)%qzZLD&1waaoxYTa;gp5%2X&<(`|%bfpR)r>{`g zZxPGNKff^nu^M1r@o8xhfe09uO&q8&7#J8JKv7Ur!-9_!up1-7V~|k+2KEj3e9*ld+l&Um% zMs=cnSxdzWeAo-L+STaLQ`Q4z7PwW1^KF2)JOf+;kN~@P&c`+oQc(J8w2OJy_^mkf z2h%=3#P47WS@mO;|7*+iU$-mCSkjajqRcZ-Zt5;Bgk|OL)FYQZd*q$2CWFTGH8;1K za=kBj?!Ljnd=P-ecFx@e+&w*C^YQ|pIXOAOz)8s3TiC3%e8ng#8rR{%P-eR{P7V3( z`Cn&CokB_hC=T9W%B6Ezx1uv2~AZA(W^d<(8W@yCbHY0g>3(VMju(qz4bp@5L zEc?sm8yFazUSAi}p`85sY2j;1+^7}3y9=YQNdZuTX1lkL-3K>(9V5fb0Z0H)JfZ2f zTK|bmd3XJjtc!(G=R>yK$Ev!zG8A�lwmn+?EL9^zm{{JbGe=9E`V+7Hlh!>MqPK zrWn1z?;0S=QQ&mmClPCa-Nh&?E3<#Ra~a{(pR3lBkdpfKT<^AOps4!?X0VGxjL^f5 zN)@voJp$V^TG8GHCpveuM5ns1*+p^)_; z-T(dY1bIVM64tG7yz|99kHe0SBR=&y>w3j$Uwd~Nl%j4}EBO)_508&$=jT?FR7%+0 zuH6vZ=Tn%yi3tMLrelt}=E28){B%Zi5lpht(a{)@KzKIu@f^$8o$YOGV&Y-&`xAh= z0~MsL9RT$-GSdL`<260~i%w|S2s8?9#=`pg6ku-Kd2g~%lZEmHXa&7qN{YZj+vEMz z*MtP+VkS_%fI)RM1O!hREsYTxzq~x`W$UkdQ_nzUFt*P}CF0H+4>8Z(*r;;6*cHLX z*(7U4@jKle{k{9~I&p{o3430(t4E-FIqP?3ZM9tAqT_3!vj1DA1uS(Vlzuj44U?m3RUiMnQ9xi zA76j}x3U>tH93@76`!kpOKZtO$%4sCVpdB0T?f!1UNP`{J^!wMZ2*B0{ zyv-_E@fGCbO9hG|ut??iOKpIskDd*tPmBP5X#aNzYuP9yBm^kzun7n>j*8demtu=s zT2c&pLuyTi@eUbqk}qA>y1Cie*};02PF-zny*knJ@Ho;T(~3~cAHc%nmR>bL{2>-T zA%X$`JpbBS%H!iphH;Y#(b;+){ADeD;tqzf|A(pXj>o#~|2`#3(jX-=l2IgCNup9D zM0R#&GDEV7XqZV7vO;!}WUqu|CuEaN*?T_k^S*z-=Xw69QcP<)AsnAQRe*5Qox|`_e4eYbuwT3EI2hv4^t1@gbya|sx z#UrVu+t8|TDN)POaj=?aw?w>Y`v*n{dJKVz6;#7xnw6CWe)BctdPoa71=v>0FkA`w zBOvsKfUu`)=Tj$3?|JaAGql8Qo%+zBYkehdaI$%XIM#Vj@6S z=}sHw=bhI#H1x@|etN+YlfAsz1hb9?Xg$8Ojth+|mR;{YemPg&xfW!u?eoXj-=Ds; zCtfWr(P6q@>QWGk;7;O-0w?e>>z>9xMa=>`{hrmvlmg@INK#(z=Drmh=|p}&5etk` zeh*NDAg)O+y!sGCy^M?uQ-NOVozjUUhv{DDZI2h(55_lR-ik3CJNA8J#W40ARjg-N z*wj5zO3I>MN79=%eCpyGFMh6nK1H)LZkS(4C{ZCs9CE1Z%WpI@M)0j#O(@Zhzpz&j ze(~uOzmAS4;Lv>pXknbO+Oc~`$@kvrFZ0CoD7-d_c5&fvM+j0UsE-}10}iNbp6gSP z+_;zm4}5WhYShmB?qtm!Eax2$aB#fKh-c}($wt^27g*2{IaiU4??)Dwfi*=`XcZK$ zR|hhl5EfpJThD>nQn)Eole6p4k^GyICI()O-T8E#zwg_Z@9+%?x$=4NkoUPO93jov zCj}42y(HoYTzh(;UcIT_moPEf+gOV;U?1a!Z*fV6S!l=LVN-}dW;lR!~$=ZensWV~1k@l}L!lmK|upEwEbiC!@ zsTtPh0;h88`#f@T@-Gu@8cK$SuA}u8buA%ziL#1f5-t)~4rp@1%pDWMjOm4_n@N6i z&V6x0RcFT{z9VtPefl)vV}vFBJSgZcSpCbDKRpp^L4(Iv=1EGiZ=cnR#$V_rdty+B zjzSJ#-j=Sv;n-ZOdq9)szaLLPNQnLX`8UYXnE?}FXx2G5Atb1djrUg6L0-7O50pW8 z_>m)|w6!|wG#DVf0>wx@Vk6k8 zNfuv4N6X8}`CFBFAXY|3R+c$02rVI=NlQ-;y7MEpPPOKPzizQTGG3ly+nl5Pm!!}x zb|$oT7U>SSy*pOGR zz7lHlb{9a$hBq$&SnUW5xn^0o?Q%dbWPcJ6?({wx3@MI zbrUibZ?JvVDyC1#;JMYjq>$FinpTvic1zP&Grm0a6AuPd>Tv(O?X9qP-aC`!2+{!o zvrg-la-z@ZS-n^;Ubue!I>IWTJ|^KJA3kyhAY}RDf&@$ae>rXt$^zqO8dvm4ougnL`3Qrj%4aPD48a3fT^MLS`*8)5azyQjO7K_K!C` zNp~5*FYSGBGF;8iY+83OR~KQVut3Sm%(p(r{Bq@yWRhFj`T4;y%u054pCI)4RzXTg zi97B-KYGH!et4~POR{>o*L_X&GtjG2i3N9UTVs`AR|nBcq`pVE6eT z{Uun^?{M&&H_0xi0kn4dKMK=zPPwV`S*2&7uh4-PugztQ(Oq4=-OOCN5DQ~udSU3a z)>5^et}}(JTg}Q)h-Pk1v=F6VVJBh7=g14R%KP_!3OaUtdt&Lo`{AAfiXUUp-M5TG+h}4GaL={<73lQu>N_JMrLe{PiGy(3LP6R@@o3|Z0BXlLktYnC2o|954UCqB^7SRIi;qO z-CsWas8F?j_e!J3R(#sQTUxAuNi84;>-{~@t3KK=wUc~J{6XQ|oGp%9S;VnGJNUON zH&bdoxe%m>!2SNqJ+B(i=oAPV{gz-4PdR1PDDL+@-Ct{9z;R=kcWvJdJo|>GrWeqN zwUl~z5E*j}BzGI$Hq*KV`1-PPawauY7d{xLR=7C36VO{cM1}>`xt4zV^CO!9v(!vK ze*ZpTZGE@+t=0ieY8<(qpp|WGZLe{duYJ)xb;{uy+3mOv{&VL_rtwXB^6j2N-A|0M znVU$Hn?{$#ih5roE`!wIk||E2saaxb0Z@;jq*)qIof&mJ&^+h;{XJ%9yy}t zms1NWE*X74fZ^@*)Qs@KzI?mjThiKtLo=ZYAr1-pkhn~mQqsW5T%Tw9%Ncc5mdS* z!JCew1`0pS6WZ8vIn?}r6_Zrd*EpQ$*4xD_^tW@O?sSjOkJs}lSqsT zE2%i(?TlSNIPI}5ia0LlYqvC~g;oNFzxr@ryY@GEcBM!4?R6||R6IUGi?sr>jMvw^ z=9l|U*nN1TqpZ!W57YFbuCZ5K}RO0)f5_~#cK#Yjp{MNO<8 z5upe?>cfXSpEZkJ z@YoJSwp0X8@4=y=gz5X8hgy4!mpy*gQXV^YGWqSnJNBLMZ!g*3ck4hz87A^7)MUjZOMJKxj+=ktgT&wALlP2_`tH@uOjLgVe!#~d z06 zIkgV#yh4OCh)H*l27rd&1?fn^!+$c6+kkKf{sJ`X#07B*32qMJi~mf|AZ#^+e7I#d zU5=3P_iMRJp4)&f07$Tii|a1;L0aC^ho)`3ErS8Jg|&S-t+UR9RO#zW#?_YQ@8uVp z9L{6?T)=Mspi^jIZ4GK_AN+#H#9iND4(tE(#|w0C4l`tjSWcDH99K*wukE;ez+3G~ zTI(D!IBrebyl{{>T-e(3Q&3QJ7ucs&UA`nP9*vr_izyyQ84O)IDhVFD<3H#Ghlk@2 zq(m{&+u03;PHs_Y>jV?pFw59f_&ch$_xI0o8UQq4^BLoWBqPQ-)p$_<_)YR0d=!_>%gx1>V6w%eYt!ajasy^^LTa-2v zqZY5m+I7%}b$ znEg6{L>kLC{pe{0mE3l*?Dm?;b6@`_3^MSk3YTQutpmUy@Hc8z_Lu&+&g}LLjqc*@ zYIJeO`T3RhZZUgmpQ--!>vJ1({&POEh--TUpC1a5$1>JhkfXsF%_JZofYikwIHM=r zh>L`pkue}8x!j>?^%Q)Xy|EeVp~8*@-qQeX>2X1z-&-)=j`v zQA>bqh|Mr@EVO-qLyhgfvkIaJzt+^;=+hTGnxgUp<0aHr22j%3ca9PT%6WMQsHqVU zNr%4Ua1AQ3_c@SBfe=5rZ>`>yj}hOLUSC^T`G~AUzLg{1hXINc|8}}?xhg#8Q)kX_ zu(H0BU&rtf_n(FzZ=IMy2w#povK~)d=O&)7N3d650|`V%pey_#AP1$J^kSQ zfF}dZ3!r_^vO54jJVN@)m7Y|I+F=f;Y_@s4W|Dnx$gOZAh-ZF|l9P4Z&whc2g_rF$D%+AaZBRPquc}A+Kfn8`= z*mZ3Asi~>8eUkX5tO5cd8Cp>24$I)E^-)X9$O?#zWWdk8^D9}=+uOT)`L>mn6;bc0 zpwKAgfBS%U6`NE9RqV%)kG#FLx=R=V&apa7CA^G|mXVh~EF#hYuu1Ia?2s)@dU`sP zN!1MvB%GW_I&;i}($ZX_XehjzB3^}s9c1?K`}WQ8WeClUn>WE31^D|vu(ivEuO+Re zztlG_?)c%uYOqh~TTefF{FtVp;X__QfxeYh$Fs}?edr8`8KMw4y;=WqpP!G9C};!s zG=~W(3mJ<83DhX4fF|Nt6jO)c)Ueb=b*8M-S0VS)*8;#Yper^ zL0uLzY3=W;txe;*YHbS{f^?!1P?rTpnRw7aL&L-R$kiCCGy}#CGc43&CxWFT7P|Mp zYg|qcIY9B~dP1{l-ot;?fDA`|CeCY@x`&H8C_QMz3i$o|w&A^d=-PRrqnX~k`Oe43 zSN?mY+<~$7#}6n|-X|wN`zoPoY%p*Eqjmc0YcFIHi5=9LGx9hIT0TTk!on|>+2qW| z-ZRpj9|E8{Fio8|_zhhWLqDd)JBEgSp`m1LZ6kRF7w{=HSC3oyt2l3uaG2$2E{ncb zO2Isl@Sel=pXO;1k^F*FLDbK^E1m{nDE0F5qxsjBTY*w?Lw7x%_zocafq{yM=lyaN zKXYbZa$Zv;^upOD^{hA_u=3|17QPQ4qUQ?{;er5r7`BQey;AqOw;~~Tqn+Nj>K+o? zm!o-8*MMV$eBW&=KPgKy3H{RE*tR?FL+5|KduP@aCL*dx_dDd}JJuI6mHm1szmRTR zgP@gLE&UE20@PS-zpJXMo}g^PVXPpws1l%&1ZPS5;&0IHpx|k&t*51)>%;&{@=Qt^ zb)y-lI?m1Q2Zz`IK)$Fbos2GEB!|Q0q}o5|n7fr=S0VS)c74ebK$mG>iI|wn(zhQ! zCOk(=YzD~zvA9A-S+u>i9<-^+R0)M71x$Fb{uump?YxFh@)ISh1ZKf%tE_IKn4Iwt zZ3=?s4)F18bGbSvgF25og=P-un0NUmF`66DYV* z4-N0$eTu^_-Y6J$VC+$_htS?J0z}An7egi?FH5lW0}3%_X6s|e%(my+j-U_0{M7)4 zQ3Qxe&L|pgbBpPLfpXF5q6+-FYGHAjTm8cjO8p-kJ(nw8Y{r_76X7m&Cr6<$2dHVZ zu`>J4k%e|zp)*6v^q1zbur2R|Jnzq8+x{x5-h;WdeBXb3_~I5GPwMn|OgoT8R5&7xu<3qIm85kKo=o@T(O}PKyfe}1_ z4^ZT;7+TBXEC`T@yMuB-YzuO%1Z@6X!(N0eQ7&M#WNXb_eEqKg0%cRhtSl_*khtmy zK!~2*D$bso;?XXMK&Z98oD{jw67lDc0iGcGC~n3J56XIaGH$dVWH@?s03DK_A7v9` zGM!*zLc;y|5vmWm#Z_CIe1E%+Aas)(9cxu}b(+S_+=JdO#zqMTkFGL^I#^+akqQXd zxw>9Bd#8-X(b@Ujc+}l+xhGLkVh1FgcBmYXQ{gmeT0 zzA?52JX8>z@H@>7nYBoE(PM*|xn87ayM63}S86u%D(T~3+CzU9diH;}wN10s8yPE$ zQs45D*#R?1W!m%U5{yl2@Td2T+QXOVshd5` z&B53m#6(G!EXp7&IIoBZNo*_&FgXbv&8K*Hh9_FnkoNl=eA<}kwK$)H6pu8!^K3%l zq$ZO&oVbZ|2!aI7(lT_*I}Q403meIc-DnC zb(1OU)gL}QjM8SfzEm&Grn#|Mcz~LpoHuhM>@)1kl?@HA%gTItyRl?#r@A$OU4P9o zOxnHo@0G^ro(RcVdGK9mVCgP8p2emo<)Zc>l7iLMqn_(OjQIZtvq-&pqPWoUj+)wG z^dZJzZ=emsB5Rxhz%@8J+CG+dDlB--oaMOnw*v$tE?W6M_n^Pgelvx;wi%Lc zo|ybfrrG{O0}ZCHRQz$|%O)z-;AFfZE1Q-TAje^d2^R~T5Am+f?9J2BBnSgmc<|ss zB;7_|u`ALnc9W7mHVN)W$rJK%ko2AR~vp&2Px`f-C?Dhmiy32D2m$@{;w9GtgCR9Gu7YUe+)Ydwx`jYcHMYP zVi-W)wx)TovkwQBRFFdd=rcApW=0!fu0=k>!ok6T*uyUt9THgc+v|PXX5F(2($f8y zIFs;aQ8Rqt$9)UJ|B3U_dimCVy^Oa0mA+}44U?lvp?e-L?0tLKW4^3>`~6XY?-yOv z{!?IHMqPAz*^zy>rNqsJeBUu?2vRT-t8gU(?;FPgKcZPIyc<9)P-FL{|kb~<2nbq zLiZqGS=V_yvD$aikGG`k_wP3#E4H#|sHLQ&h)K%mcAzl#Nlau%ay30yXlmy};yIZ= z8co#5E$c$u@d>9CeYG5r2%jUmNdDUa5h|R*7|w-Wb9M9J(8Ux)zL)Nn<&uo6>qQz) z`ETD6-q95x`W^erCWo~V!yak@^}&PXXx(k>?JGMw>6Mk0vBVWE$&EsZ-ewHF$izO2 zdi&Lz0xeMc?+-l9+dOF|5{XU$=0Ed{YkKN&sYB1Fp3t>Ncv88#q_y>p&>x{bj=)JjLyFbZlbneR*(zl zPb@F^fDiB@LFH&`)~g$*IHo>&!&R!1TMYXpaeR$kg(RRGSO2V!X0cF6ai*W<_e z;E6$M@Z(+M^tKuJO4f638H|M?BAQU0iCzqsQp7WWzNsoJdjvH72|9X&CaDpxDL{4w z(X`{~YhDBA0Wq*ZN1~#qCkn?jF_vBD=X)*Ev7Lq|vV(Xbi;v5`x^ zr|-(P*UHIh@w2ZcaXkP(Fx$#|qFqQP> zkLRWBjqIt0QmOcq9LN5ct%an+qQf6L`?ozV#R{Ee*{mEr`@N>(q&P{9SVoSZ38Xel z@!Fsc$6H#-x8?VbWq@w&dN}(>5|5=31vVcaIim|xK>V$fyD2H-!7NcRUBwPT?L!fL zERH;ymD$MbSFa915LF8lj|67_WB7ttDLxpb#Mj17ph|oPP1bBGh=vWe(55QroQETy|Fk0;^`+Tp3XtsurV$qpdi!^H|;di01FM?#}k}hqgs8VgE!Euj# zaym_(Z{Lo!IGmSYK)?MpRke1!h4WeY<6XHCdx5};ujXISNvg}FHko2cbDJ0Kh#T`8v@}qH3fzufk&^d zkGCu?8vr4B2&vZ!aC5q|cL<;n^|v-wg~+#|j@y8dY6wUGZuXeNMHJ_L^fPmjn-YgS z@K*u08zXfc9U8(bnf%1`hV*op`pNGMeGfJ=GP=LD;S_72arxoHbHoe?9K3gM5*N5R z0H{VEYTivzYCG(EM$vs5Rt=jROFKJm-ydx`j|M9+O#eFHHs7dTcPGs`U|Cu9`q)&; zCZ09%o7+$PdKz5_lnEIuzUcRf!+4nbqVIW>N0yT(lQj>Hq`m>KL%d%kry8nkZNm{+ zG*q6KUW-MGyG6i;j-C3RVMG0(0AVM(z`(RL^E9h^n1m!_#ET*E(O;P<|8B&(d_pja zbl1c9=H#R2ERsvE-V>y)40|#&-^dM>TxDy{(zx|a^m10jYYGA$x&q{QL_$h@ejv$c z0V#qLczHAO+_>*96AZ`LR%#!#pTmv}{9zPyrS(>eZ0(Penc6OIgg1gHD{Ngv`IF>7 z*dMXKcPjGz&5F}3g!v!6b%6?EK3#R;Mt^$)Hb3eIFz2%Du2%zI2VXh2OybKoZ{BEz zyklZw+89^xfPfc0^CJ$w44!=(X`k_estgf~AS^5lk1tgj5yU4FqY!C~gQg>18UR%e zxD28-iFlz009MT8utUCY+nR#|v(l}L&>Q(XD?9D>m&lNi`KFatGy8zShBthVVX++F zni?5VGb~N@bZj;E_Wf1?-`)2>Mo}Wf%FgbC@*ivPUN01M1#nu;jJ!LRfMT4+6T{A< z+G9IqDxS~#{P|Tv0^84@KhX(fENQpee=C;cHFx1r?FmG|CXtqt+nKyLGSy@MvXeEK zK)}{`oMG9+b@Pjnj}JRgcE3K4KBO}h=~kcRf2pjyb^QrHH#c**+=c)((`hcQ2(8>H zY@v74x9CHkKZjbR>O|OdRac>VEn&|d-&EtV`FT}rb#XOqwNTRLDvq#P3MwjgLBUs8 zBrxi$tEJyo{KqPM{J7`;U_{HyM8p(B_8tpD-u+niBLd;r^?-Mq#246cbz^RrNLgEgC_Icw15Oi6j`FKwvQG)xIy&Kk z_o(;yypzn;;Njunys-92x53iVl2XOB-BeI>Zr*QFrI;yidYva!#{O}yRZjs1tgk4t zxP4?FwgxeC1B1A!=}Cx)g^;oWyox4AvmP&f^>$`;OA8GUz3~fe!TGrdtabK0Qm5)o z(=|l&i`)8A;>V9Kz?z5S*HL(9F!Vi;B~QhX-#)&{!_7SiXWd@9vyVWn@mqE0nF$k6 z(m|o2%i;(7q8W7X)@qby<*#20aibXpQ^^^Q2Ii$qZghOMq<{XrM8|$Tt@X?A{oMt5 zuB-GE6b`pie1oMYgh_G7(38J^9}tw3v!!p;;Mxlx7n8rcC4^3$8VA=J5*770ARwot z;+~+5&1`o`)RV9<(aOq2hR=}^5&HM;QN4N@mc)AI46VcRx_XP{hp976Z(h78!@>U* z3-Pw4v$JoNo0yx|;;>LWdC32$!25>Bts`}Hb?Lo*9znI$)nVVh zrJ3evTJ^|CCUOCPAUzTm;jla%>e~J?gNr`k31ovkQBKdVeMM9OL;yp8JlNRS#>gwt z5W*tu_5Hgv0uQKHakqmg`lIYIge}Tt+2qmyU?`Km}Ggv8cqo7X9 z?!3^sIsTr0k3Zk;`5!yAM~h;A+lMFyAY9+S^WY+-cvX&0l~mTAFzf9tx1&O-?lJ5* zw21jMc?Fs@M;vxtD*aHjwN^-V)GWl#!68%e;(Y}3U~C{{WIM)oSCU<2VA%6PyS%$m zVeb}M9MwrKE>;AZBr2t(?epM8;|}hg_yU;33N#iXtfdje=3`K#+pyW9kVcYc&mH>p zEA>dd*6pP7ODcPJ2Qi9D-MaN+DhmiNv6-XfEsAxqz?qbOJIjkGj{-d@6?lw7<@0X_ zTvY;1fmB*6KTeCDR}0m%NVhqw|8?iruU`QntQ*?5ucCLsFrKkgyauTZftF(LiebKI zX{kq!+ldn=%#T^4tAw4{8)Py@gV{2>u8s}>gkP}`->nDseZChm<$RWn&0951%?AG| zxt1?%-Pq`M_ShEt0PvRY+_{8wx%wifn1jbI zt_+B(0)#L`vNw^Zg82dZf^Rsp0&tmtb|%N=dgb+wedd25~Z~3E`Ya7xWW*4xh6F(O3Wn zE1R3Ms>HFS^+aDicgG&W5)vr&NDqv^h$pEsB0vOz$W>5PuI%YhVCaB;VqN?`x5GZ0%@GC*FW#!~FJKPRR{QTJ) zMT3UMj?TqJ3n2&R9cNB=efjsbF5TT+kLb8fYc-hRy~vLa)9bMd7M{k|a_RJLila~A zH;ImSzNne;8krzjhSd)V!*I2w@0@(q<-8~9v3G5kWGakYzDw^iA zWuSRC(Cix024~QS))Dlr46#z(M7NR&2A*dOMuTvQIWw3 zifiQBdl+MIks4S8VoC(O3*^akpQpQWgW|42>h{pi&Ja{-(cBMi;wV}mwA|aQovdPgEW||m6D%;xLXk>F7JQDW( zIn5DxY~s~3b$9N%r3X94A?p4Qzstf3)tOw9RpU7dxio7;F_sfS-$RVs)VNOgAY z`^zb`xX;S!AH|*t8K-=R@>X9V^Q}2UE6JI;6nz94<<{eTr_H3b92J;+ETE@CIv){? z(d0H#U;OHhyS?Za`vMD+pSkI|KldyaBZ%0Fz-qE@`BXZaovnn4bz=t=LmobhX{#C7 z_(VtyjF?5A*vE)Vvr%KcNrHi8;F)^O!Nv8m((h7LO-)wKw7;1ktw0wWHHFOZnW4Eg z+V8ZQmja}&%E|3Tj|`QF4FG@oF8I0D7ALC#RE6^!%Y@M0u{`6xX-|8%;g{}-#8aj@ z^eBfgk)=B`BO=BdtvwD-HS@&P>bPc8rrUg_6YEn~vU|X@VpIeHJuoz+lKD*u^9`cu zKt@blpz2P;BfxNEXl|}G8MU#_*aFb#?%mPqh%97o5d#My$iRPk?z@t2>T-Qm)qAP+ zvM>Ensz+A-NlW%TX4a!Rgt>lryd(PIzqfeeM7C-e2kC>c{+mESh*JA6M|NTJfVD#4 z=4Fe&l4z7?mmVg`9$`*TPuG^Wn)5>;kAWRU7b(IShf@Nm$fsx-87&0q1vPod zLq}_P&b-%-FaKcMq}j^TKccP|mr>-RXn)YCCux7K$Hu|s!nT)tJoavFZi))r`^Eh(9G}tso7F>74}=Vjs$f zY6KP-s|<0fyjMj4x=iNr~UAt7~M@a67nIAr^uW!n&<$8mS`Y*Y< z+|`@9xq%DGCxZ!jvUYSESc>4$9|bj5v^Vatyxtn?0scq3@L@)qWoqY}Cq6#oT*;IW zA?#!&nIYi5K#)bAQ2f7$j8uTKF$FP09ohQ>OoaKvNg5jDF^&`IGDK^(wVeZ7D|DOt zNQRm$P!|mijfAATxSUqr+4%x4S_DN!dgtz4+3LXFyu9CoW2Tg!{UtQKz@=fZs;sC8 zefg4_nnwA?jhdmM8ejI)VEqsDI#slmtZym3lj6|E)?-_wo65>E1iq!wyLpKOeHC9 zrn73i6r`G|%l1LHK8i;x*5?qP#U6+d|D4RWRzF7_THySz>;dy(Ic%P0J@3s{o58lsE-qUB*-Zwa#uI76FTj&znDkcG9WH06 zg@lLSbap=6(cy#^AOPU0s#YNwoIuR6BO~uI#v53iuKxa=aP?}urU{iPk+Lq*z6sz7 z@e0W*l7|ir1xH3EaeI@{D&MxS%z*UcYeNGyp3Bf&oAON^(?ZC$!Erv`$9NH=+0f9? z%h#_ZA3hX6b?OFgYkfX9cgg(-(X#OIo&8dkZSAq@4FFYh{tgjKE#?D5(NcyIk}KUv! zIdO!0cTOU5p}jYTRXdxJUqlut{hFIGrcde zr9Y>#VTC;fi8Gf+wLc#}|JSxQ-=%+1FKPqM3J5@_p5DvI@BNN-| zUc9L7usqdqH>MK8hvq-FQRk?hD#l+rA+{PKRqJBSy=%aVcld^48R`$5$0?enmG}tVFch77jKdVZkVz?QZ_v)!YO~Jr!d#5v((dv zjFGwNa0<^$nev}M<=bs16_ZCdskx+%ke7}nhtPZ^s?SlCWeMrrKNZ8@$n)iGn10qXq?_HG07=Dj?dz)mWG*` zNPX=WU%A{~;j%c<3I)uLR=-_PHb+7l4)p^@4w8)5Fa3`2-M1l$ZTjkEd-ZDir%#0B z2?a&9RlTbY4s2>_2^PN}ZO#l3zRTSlpJ~(juA`l|G@kT8^JDBOJ@@P+HFL1y7mbZm zuXj#Y1*8QW3T}Ey`+vE4%y>BsXAs=*|8eu@ZdV_`2Yme>=LJ6K^wb|uh-M9N=#7W8g{vj=`{tJZ-MIb5|LbHs!@tEGqv2s=I)+P~ zEOHrVTvntcB4gFkL|j+y2Q;uagrM1^?=vr z+Q{40F3w=oO0*VhJf8Aa#sbkx*Ep`~sQkLzoGULSWqXWKjHw_>DxO0`gH5UPLV#3! zWOFu1Gs5vEu#q$A!*S zS@mBQG&cUmtM#PaJX7e-w~P?jtBQ(7=>NH>+BY5Sd=DMTC&-haCqm~h6dfj$Sg<3e z1+Zv&`IY~!eQP3=28HHjNb1vQ_itF%)2}SFTo(^)?6NleGf-adu^ssi_{(4$7z?Ak z3>gg#ijquQ`6Y!oG_p8{{1Z_rK*2jH@bUvg0aoN!IiI9;_CVC-F0s5-!HJYeLOwm zdK>zF6USZr_BIBAFk;#&gNKI)Rwo`SZk~>Ne&P_`SpKyg#H%28_%AX(Vfn0yzt7Ta z!b(f#ArXn%H_Bcf*$A_O$gZ8@0BuMOp0J;?VG@vYKvSPx)g4t_Y56 zHA_=J7$(C_zMZ6?F8XRd&*o5Xfsa#l;LXGC$~X&Y1_mf6H>NLfbB6#5*vM+gzt_@c z=HW3{6SuM|H081}naQnD;(8-KdhiNb@A**=`CCl~iYuGw&=e5~OVoGJN38`nTVuDw z??#uyv9I{&ynE6kdv$(>;onsCm$?SANuaO0dcgZI#PP9nRz3jyO!?^pr23BUwvU=|=1SP_4`&F%d&tghE=Xhd}_jeR*O=0(TvF{YO! zBB35?i+`V2ecL%-@s#An^7vz$#%QozQcFvwK-_D-_piN^T+l8lqR_kCl0v`XFxeL1 zzUdXY&#=o| z@H~r8O3ZmI8oInHWCFws3WRVoyFV-V_%W-Z)sIp191YXfQ4l`F{hOlFX*AkqD(Je{ zs-CRkgV50CVwX`V(e@`)=C?~Oz6EE1UjOm6pAQ~?`1#YIWX+5M*oLE%h=9fKked&u zp1ckXm4uuI3jkn;GIDbb8Nij}4s~x_zX936%_ZGwh4c=IkNJ}eEk~`2nuwNEuO_c& zI|;~Ky9_m%#?ihtbajbtE-r7Qy2eEh#RVJ{ccrU+9hHPW+im??D_3|t z`n8Tkp5P*{);mLyq<6nkM^98#^l)8FAdp-ORg1r|8VPtcBSFbBhWbUcs<^A z?uIAZmAtMp&x+N7r`G4~Cs@nNGv?)eq~b^ZcF5u+ZqB!xjQK~n^+HeaZ+*{ZP~%PE z{mVZ__bgB=+ugZy?|I)-W&=}m+3t2d<%nR#f2(MI{v$(1)b?52|NcMvKfiU0h95ur zCo_*64<#=qgVDp=P4$;fl&3{ zW+J0H)ZhON=TN^@UE1`P2=$#VS^e@SKUyr+QcF!u>aPza=-Hn+TEfVtTs&wSGUGNaw)gbu7t)EZ zhBx^Ve>W94XM`gOhlqbo{JA{dXOH^=3B=kct7F$zaE zdRcKdjBlmOOE(*`*`KJbziMerQI_@VOXPjpodeC!cM@o6?fYD&l>r~!T$;Z0Bs22} z3CR}ElvQ0ZKrdly4g`Zc$=$49X)k>ZWyksqBU#)*+zSv;RHTh;$zXqf3jl-qR=pd% zrpz2XJRucN72=g>=SL!$o6^)@lw39A&n(%xpr|O?@!95t)`=4!^C6yab#-%c{&A@1 z;Zs9{mD1cTc{7(g;@@r$-PA1yleOoW`hDI@c>wKZpewBRX>(14dgrhz2CS|ui+OF&C=RK%7I&l01F zft6M7n2ngPXe$4&P_o^|9r^WZQy#R7i#vC341U#$8U=SwDg3hQaCS$3a1yPoD5hTK+W4VPj9U-P)k#7qA;^O%n#AK%u5~ZhxP~ z0s3aU#$vhZz$gIHhd&zET0u5}NQg?@>)013_9;DG+rTjM<%<{o@$od2N8J5_6gY7{ zCvF%50z z+s)1_uq9gy$iI>5{5?m7&tyhr~Kv* zQHI&k;(D5zQU(Tl+A}$!O7a9X!&@}72XefMvdbRk>WekCU6I>obYyN_W+`4@&>t9W zT`$O7H|D`)F1J6je1nXVuwuMuwl%wTu`N@py@-aO$U+2S-Hksu|dy*hScW7^q+NCW>D=pmwO@QXwfX`{g^ zs-APiv;_cxMT>1(dl@gh!XQu1!!!nH83j2AO(3(|f?{!?;_c+Got=T^?G4JZM{;+` zC@55v#TtKp&vWu*g5mW-XIiR;@6r{W1^1?$8~=;l-N$|-zx43G$QrO(TZq%?8)Xxf zprj&lHc=O7;nXA#ii@LteeYJN`Ub-*+CyKS6>h8k3bR&=Bsxb zM7dJqsrqNE9Zty}IdH{dx3B5n9y%)Dch8GizWf$9Y^QSx;%jCz^B4MZ@->~%Zx<4k zzhvCLWCtpDuu5JyUAVrPRZtN3{N2e0yFkuv7tBHIAO98!%m=c#7F{E-P$tTXo;9pu z?k;7nTKdO;!}5Tz^%Hm)&<8OtEuA~WZKBiPQ5P!1jw?l|56&X9F6IEk`rhZ)B!+*q z&diM6{MSj{aJ))VoL2+Ru(RzP0;D=yUt{R+gM_mY!7{#uxJ#eMhw(*2jSN9uD%yCZ}QCx%Y# z)G5`yRqt$h!)2-HBHj4|Kf*>k))qBh^B5R;>n~1R7?oL^@No}jy|%s{V!Q4Br7=Y% zg&O0U%!8U zh8kC5qa$NX@9gYU+r8C)LA>y0`umQa)4FK2`Y2I3B#nwtuFytTDHEf*usNVtBz z+BIcfGRXroDPnCzYxzUDRAGU648o!`S7YC5rOhID{C4it;XAUu{(?MIt@DlX+?kTT z1enqDsYxCkdv)etnM2=AlbQZ37-=I0tKD|*K3{=y4qW9;WTb(y@$c6RY48-eO>3ZX>T+g-I z+`XruYIackt@yvjaDj;ZMwmj=GkCH-eTtPnXm|Q_9gr6%&g`9seD`AFa9TPYoR@d^ z+V$##jEsH~!JfG0rabO+PeD#qYYuy&jL6XX{3R(s=ZsxnvaISgaVm<~4(;xHFkRjs zcXg#m|04;7$86$RF14icqj0pR#*vyvp`93BoW1LbJ~rRw)qtN8dKT& zNElT-`Ll0DqyyOSnOy5jby;;GoQiB5?}u)42un$sNGIm+676lBtos4oz){hMrwQ=6 z-PR~}YKVbG0My++WI9z>`W?wWJ^!wy59WyX*4i(r=B%oh$49I2pshU+Jc_zS1|R9`7EQIZH_K-JB1`Y-v! z=sZ)R($V+tZHH`607gS+gH|gxDJdvAdM_zy{jL)T;B~!~__I6dmXWFW=_GDqhz}1> z)URKHi2H*hS}M*_M&@N(vWNpmBeh zA~dZw(1(4{reHZa#wBdE(>x^6^S@Z7)c0Xq==y?dshZo{Hv-2e;ZHR4y3ZdjS9nGv zMFoNy$m}o2-FgubAul6C58Qx_GwH?4m+viuw9h1|oZ2QNC0uG^c-bPNph|8o0E6eBaz{{p3K%<0C#{M^rFPO8x{bnTM)zK7FG!)uEl zwWqT`3npp`B?-HK$g#H>n(MxMQc|k-dfYSj!O@}*(A=7fZqJy+Z|`Ti2*fZ0nh|0Q zgj}B%0g^^A4C6h8!eC2W)6RM97IzVv-(D2eczsIWSfXO?i&3(Qg(&lTy9`;e`sLNW z?6}U%Y?@<76-2uJdI;}e{ziQ;OfLIJ!E=V`>zuY6=hypak!U4&_4&l=-^iFo@k1m4 zcWiMhhZ9?{^hxEaV1DCc_j(Fw2;<}9-cO&-&J4KWQmMg#iu=*x0T8C6kS4Kla>m~( z(tU3LKSx&9K@JX+L#GS^9$!`b`&Z`Hdw&{s*$cM{9%wWlJh(^9+?+vMN2`CLb4L@~ zTi<7n(vB-)Oc(R2AMbK_?(Zt6t$i9b{^hGz(wdr@XL3r6EG+z=KR-1xa`<;w7}(K3 zrNkfN+oPA8->t@2WPB9)=u!G*ep@nL>cQ*+n~+dwE6!Dclh1tSN8^){s&Vr`N?)uv zO|=5cSQQl^Jv}3QtwJ!hJz4Pu??9AI^f{{VxZ}b}3Y?4Or2m{A=h6k|eEb*-$`zR^ zeo&y^yGtmFxHtpm-btRx)Y>`-de|lFSDG0n$O8aX*bM6CVy*!HT(5b3lT(Pdr^f0u ze{O`(rHYzUqk(P+JqKZHz1WT5N;A>`5vQYht`5XeBUv&2K|`?>0r&evAhzd|`=&EE z+3ddyVO*-d7;-=@UxDd|vi0ZJuB?b~79~mY4&!&plcp6T12=ETDI4t}STOY-YT8NA zb&ZN*e` zzUaR2Y+uv`K7$043#-TVn7yXE5xmSex`%ep#iu_e`K^BKu9^3CxyLU1PEe>a&$#w< zE&X|su0IzGf zobIS~W#Xz@%BB%4)%nTZ?e?|2JdK;@|tSourbJ! z3v{gSAlTH5zshwHxf)yN5cxtn*3fd-$YQ!erquANTc6N}CwKShrl@kUanVkUmgf#! z|9Xu8=edR3OOi)>6RE{CdZ}rPdBnuTF#mlUD`qLW`e%7^Gk@!%r{4NL#?Mh61OYn% zJojv?ditrp!Xz=?kecNFxnRqXlfidH40bxR%kGHCS%!D?&F0MGrQ3;0<0eDuY5CAw zB^3Q!z0O)bH{Zzj>0|7@Y?C+P!+nkSAm#<4m*9ju$ z&yQjGb+^NFgyH&LYpQCX<+JBbUNL>TxZ&f8Qh4sc>7u&sQ>KcFLFa9a_TOzdRx@Ah z)WbMnPn6t{kdhna^9*><%)&(mlYa$B(vEz)yYD@B6QJbM%(V)5`m|b$Gi-zIt@A6w zBbCITXR2KpmZLdLE*xQsx@Is@OOTa&IWoE_d5eYbX7I3EZ1~##nQG6=W}=-|fHFs8j#f0vQM>y{(!l*97qJHtOBMb%HM9V+td$N4$q zZ{|PTz=+;;=m^b_LurI3cZ7Vtjgx*yA51Co-1+w;c;*GinXSAyAUs{ z*Dolc7>gEjk<#wYmTQl$ymzi@{kGI+SrHAe+#s^RHTndzzF*9RD>Oj@cPtwuTIiF7>+PZhTHW2qAAhDBq{gq6%TNghz~5@D^LAqv4MQs*ep3{ zZF*}uSyCiBr>SUHF}>UhZzwA%#SH7KfvnMU6VsEW>s1t9GtH+KNFLp|%JK8-U-6Lr zliB^UWaN_UDrQrfP4&)q_Z>T5_L5yT&vBL&y35;|no3y)k|mo<07b{*1eC(mgP1Q> zb#+Bzsze9+f>&+t2KkBbh)_@d&R42@sqcNOcq(_j4!W{QPW3y>P1L+SJ7@orN%i2il1L-BG05Hr#KWdC-P zI1|>At|0(*!Pu{qkg2_|6|UXOR`&{Vc8+yi&ZnOZm$=5d+Z5!{tsdA9nbaqz7M~oT_R~HGi0P>l*lR(vRBz5L`hcJq$x_t zh|Fw4W=2Ls3fUtnBMI4?GVbGgf4{%`_uqX#9-ryWYh2IkIA?6WBd|YrRIv{om9hYBWOXsqB5}sJ%|U^8F~X>aXH<%aRZ4C06C+w!?KHe9=^1$ny-3CQ9sAGf$C;a*V8n0 zvzGR=1K&#`w@>&dkB#e>xZcY@yD?|~QNbp~i+dX4t54X3BONX|#Z2Mxd~YzYU>dbj z%bTF(e=bA65^Fa2HJihGby-wCOIa@p{72NsWReItc{FnG-51Vw*J5r)`q`7+y`xd( zE>AmmZR+g1?X)nJTz-0Bl|!?uBmBn#kt9=p8^7Sq^;@KTQDs&wH~e7J~#il zvoivL5o~`77J?8h&!e9Oid#|FK2}yW%vq4q2H1foZN|>7P|WI~e#G3~ui||ll2guc z?Q8pS{}hb^&5TB3dDm5yasQd;Ix@dA4(G-@H2zqAA)g&+_v#GY?fLkI^^}!8Qoa!% z1p>p2o@P(Je%G;hE>#<-K!{o8gI;LSH!pgdLI43#<5}X_Fyf>FIwGcFZU}&y3E$2D zh%RQ|7&pcQHLpPdKSsZ5uiFl$gUbIIa1@bW!Hv)yUGB9E>6>Lym^Q$3p{dZ`9DT66 zhKJjWq=kfnfJGD5?5Q}o$V8)!(D2695q-x6NE={4d=Jz`m0ikW3ZTdE<+^sxFI!hgE(`qyGr}0*4$@SgU6qzEw9fH@cnVordRPxG|&BbDR4yXmbe0eCn zve1`54xV)!*pJ}a1uKQ*bk`~KxPVfP5xFR^dv-Z#Nu9rJa6EfVo`LeGy6)Pf+u<8; zOtQXBZg{+R?_j{n`1r&VE<=F9rF_AT0)Z@&YAWVGcpv)7<#f?=i^<&(Z_Qmu;ICP3 zU$QuH+?)5`2f=Me|MsZI(;kvYqGOKNyVY*>Rf5T8CQd6R#$=c=YZyf)l&g?Y0s(U^ z=m!>YODU1GqB|XlI%14e6bR8oZj433r2=RLq!$jOwjlbOVp8s4jJl>h3tYy$aRlEkoD{kG{Tb??;XG$BmsU2tlW=MR57|;>lJ8S>%dcb|(#zj)Z?EUIm#twd940FP zZx*7KZR2hf%x@0wU^w`(g8i1DsAyG3;u=Y}xep5=7V_4q5%XV#wprv2R0lV%-4*}h zMVFmsbfV@adjj^C_x2ckC`N?bDep#fu%{E)#5r3<_!; z9=fPvbQ)^#+#(_~%=yZn^t*Ce(pzl^MHtsdmxlZ1d%Lzpdi|? z`?L3!&K~F8c&b-x(2~M5CwA*?ublXRSu^L%#3#3z)#)sG`^7=}sJ>zDu*-pVU7M;B z-&}QwFPY!JM=YhZu4pwBMtlAa7Du((lIy!FZcu-tqPuQQLn&Jcv%hR88JKnWvOqzf z!)FWsjLkdwD0F*LO4<$`IB*yIM|<%3M@Oj8ga$CFSM0=#sqoqyDX`aw^b(%JBF<6X zW!GE2zGNUA1L)v~ojLK-`|pf3{(>kS;+LpC`XIVbdjBmXp)zh`jRpE{Um-Etno))m6L>37MNePgGV^7{v2djr! zJTLb7^hxoCx9dK$T2UTrr~3SLP)`#2CUBd5D127=?VD)txaKu6{PiXqJtKxo!=t&L z#(DKi-cZ?+%%lX?9>WVp%D~|+vqc{}Ay^`Y8ih0_8vP!rREz#N*}n87A;D(uJ^CgV zKVr=N)D+}1?#`83_Gp$d+8M*M%j#~CDee(ui8p9*9R>j`8*3%4j=95+W^>*4e$JmJp z!63}Sz=W7n)YWbPF&^7#(-!2cmvMMy1}ss~d^5jvWNndlYz#0_(DlnfYzM zm$PWAKF+gQOm8*h&+Vcguza~TRb={kaTZeda{_Abv*f88deSvo(sZ|#_nq4#bY1X7 zn+kPBsiI22LlJ|I0W3{Og#*iS2frER?(*yvxRqmT_jQ?@@|7V!HY|DH4oepcgJW67 zPTzAo`{s27GOhBr$l!JrjsoN7cSmlmyN9&b(lfM^`u>wk6aD+QnDx4VQSoi%12qom z6_AOG(@ZU#@%Z)<=}GV4paO1@MNFY$n&CopTWRUvT3q~7eSy-LJ3CJNtTj_l$2F+@ z(JFSbj!NA8u4Fs&j==V%{+Ep3pPKi%U$|jS{jsA~l<(?u=j%@RMhiDuIo}*(S>N+9 z<6ogy=Aw7I-2$tVcXWee<(uuD?z@(S&L=hB zs>~Q_D%;*mE4`FD#OhlUc3ALI1qGL%Pj0R-OshG;95R-4pPE^m@iO0GzS8j~Ts7*1 zIcBZO%A{0&d|$XRyldoaFi+39$0z(w`)c!jnTvU_Rr_MByduhF>A{PM>F2VNFsBvX zwcC+!e!-(vi4chc7O^z(o)*OcnX89rQn%^lbw+kwf}YuD8!((Xl`VOvsxn$&!$r=J zngR8x_Z&}@)XQ^9r|lj()PI8n-$dgfs*T=z?sV5SLU)*V8-qZExOaXnJPotmSF zN@bspc~{eDQP+Gi0yL%2d)4bvzpRKM-bADyh;Z58l}YMalGp5!$s z@(@EM_$0o}YH*?#j1Kw0rD(~L8@ZD)54WzW?ug9BHhx{?2yH6-!;4uS7Y-?-y0N1x z_Yl}6U|b93H=%XsRg{%^w34=>=>i4HyTAi3ufjQ059^fkBW-f9QFc6j=+rKFLCe^r z-TogT+9LP)!^n=mm)r=PnVI1qXT$_O5+2M)ElR$gdw+mR={ci=G&@bm&66iZxz#vB zqMW7fJW5~gUC*}eJtel5G6S1wgG_vp)1#H#fd{e|MAQT8c~ zXG#t)eDX_Q8+~5sR%rahx5jPIy4QY$!CtP1X zcxK<%CaD^=<>%M24m|iD?+d1Eqw#xqNy%@3Hb?_UrC!58Wu%kI|<}74Y zE?9r-bd@nun!mbod=!_BNfG zkEUGtejOsjPTN>E#d_K9z9_`IS#Ev0D3jUrHtzx#3*%~zi@TVZw)$ALX98@&la7)Q zC-?rTXcPBVf1!8(oKY}8ViJ3IqucWyY#Ixmn!wNZb=Qj{JcBIDe^q|j%BOT|ySeQ} zpQEnyf>fc~(O?!yc@4TOtvH2bm3=?zw?bH}u<)YSxHex1$%gjPE!g>7s1D{u-3%GL zm1zA{`@-fd@w;2<*07}y-WK9X@G+Qi)6Uv@Js>S)_Cgrj#&_PHB2+$zWbtlIiA*&T zNri#>;HYkpnNID4M_zjGyE|(~E%SWW zEG1>|Ww@ozi^BZt3D*YxT6c<XZh$Fr)@* zXbSMqB)8d_nTeU14;>xh5l1gWCM%u>4of-w$r;g5EX7?gdh-+zMgy55Ffp!Ab4+b>At9cO>b9x3kn;5xI2pN1Mx2pN0# zNz)lw*aEER3#WD6zIjthQIVJ7LzDHFfS`Hd{&y8JSu!RA!$F@fcJ>gdfTUs6ZQo1X z!yvthuAcC@2|uC7O0SMQp5!<3Yg*|08lh-7=Jk$^+99hZ-QhDm&bG+wPE!@B-^<8YuJh+ z9zsUI_B_^~2y8Vp&;ifE)WM!(SN1?=0GOW^cSrZPOHB!za4@aFfHHdEF^eN@EwH%T>rfDCG?nmsu zYZH~R`rCv)zT(cwz-UP^*Pm;5K=9zxcIlv}LC1gISc{27ad#I6r|H}C6`wyx>*ap5 z2sij`G(FYnqq+EZajQd+0r$7cM3F7KSDy7Z72L4xuXbM44FfRw>DJSSC9CN-Tbs5} zK2?XEbR96Jdd*|S7aw+3r$oo4BwWIm%|l=66Q!c{@{PiYas8jHceIo<6K1 znjw}<`V3H2LBE?F<|fc@&BgsUx{N!b5hXt$5j;dpCN=_L6?b+k5?y*_`kqLtz&j8I zmpy;}RKs|@8g^!&*{@CyJwA-hw;Q7csGi?{&DIlGrt7Nf%J}DU?avWwdo}wb=O)9c zmK@K1lo_g&d7*EB%0pT%S33VpdALi~ zuFIOz@?(b#j#yVS{QeSE6KzyN#baiTDd6S1T15`)zKfr9@q{{H%le=@YEKglX{p0+ zJ0uLY(O#QXDidAv3FH7ura6zK1)bF=0?SzT| zkN}i#`8aj;WL0KdreSuu)>4+se^&!px@R~C|I!2mJhbN<&}NHIcy~|Wz+cAp%*<~md_2Y)c))=XqF~hoUv>8iBFyw-9Gt4@1z%cGlc@w;FC6!6QeAzZ4|ZAfHD@m9;_o-8O~ z-Wc;p==WWPo}+U&{$5+SzTzZ3z1erdl7y7@FT)eg)3u5}b9U?qQ)rhP9y)x;XGFqx z!jDlZ)^qHR@g&dwh|qGo*!~kX^~7lf>KIc%M{-ssy>SNAbrpZ^5)7IvH zSpM>1(VgGp&LK8+Okao9l$7`m95Ca3IMK^<+4PeSU*+avMlj#U1D=f^c(WTLff$O| z3S}8pv;kIyl%fOoKg2TR_4;p03NqAL<)QGy$GaC71KpdLx z_PY%AS_<8i38vbAl-wAAI7masBY_iWCxB~3xnODe(JIFCBeYLaOS>@F|EDdZ2Is!J z*cK*^RtPnKYUVSB={LUy!srNjGG&f`S)C&J8)9e;ru1heJws2o#~iyh@p>h1&uZA5 zN5rY8{KAkh8Qsgj@lNoWuwDHB9CtW+H;;UZ&&oQpdecbeMTf6l`n>5Q))QG#D$O4< zSVm7+)-CQQqbKjaL6qn=zha^$bmj8&W|o(iJMyfx3i@%Ef5FuORYxv1Ho~tYsk1hp zPTA`(U%&1-cre`~>l_ax_BcCQSW`;3bY$;Qn}Dq|jspA}*H9kyD+yY6 zvN1&|_4Zt-|7CqE<^1REKKt5lO-j2x5_e8N^7Ef@ON!nm=@?-NS9-0Aj9vdN-_d#t2c;0nArp)skTl z22Gj}!eoS=*3!}vczc(U957oCQ;eZV_nE=%7^v^7MB`kdwgxu$k-5rzS*V`bWr^p?2I^x0p^K zZ45r3vTut<8w&y>Gf-rr0`a9e)93^!l^AHnRQ_EwPa2UMU_=B;fZxsoT7_-_I~)fw zCb9R}l}}hskfkx1`w&(knz(zGmJe*c^C&8WahTR5LC14<(!BkdHH8n2-{vN~oYoP^ z)DLx)?R?WMzxfpV%QAyM(ZSx$7xE>j-ZGLr8}EI(PLFX!3!u1`hMW}}SgjQe6OTDLnZY=bq_V4$_ZH)O2 zb;{js%++Z5g0}EUr$6VqM|aWozH=k-W3hL%opY+@=czh=Z(Ya^Jh6LR;^Io%j+f7S zQXN%Nwg3Do#b@Q<6%6klc-d>My%ocbfL2x;w-^>V`3r5EGo*cU%g;}+X7#&5wpklw z)Xj}r=#w21Hy01#bor_e-ltS>C*kw^SencrX8@h+h@c>qNJkUL{{4B1&!%i}zeD@^ z2^2Tbu$r8jdV%Nenpss}ABaO|b}`}eHlutS9e81c_DM!Z3!E?~BtWDau3nky3nG_8 z9qF?Zf}QRj{`$6q?aS1&>of-i&7){Ts=qq)MUQ1tLPEhHz@|&0#^gG1fW5w+o}AynCmSgqX8eZ^<(q)i zTl?jU^P^Uq*?k)*P+H!C^3@j!K2l9oV(n$3z*qk{Lq;`;(;G+N%jC+rnsOIr-y_fl zChBEn(Oczq`K?5M;L^;ltyzw?dfCiL6WQ1vWo>-VsoJ=3)@R-OPD@70g1N#zPT6n2 zpYdqXN$K2d_kHp?!D)E%sC@nF<>`^RondRPVm{H{iEopPRsAlzWq!ydToj5otN|^D zMY>GI!C`M)+>>A)Qw90EYa}I~e8)85NuKXyyNDvDAh|f$?(p~TU&uRJ2e#VeA3Mn| z<44a0y~5_A0UTkmw#5TfTjD05Xa7PoQR#}9TR3YwOP$k?PwFu(c>&4;>B0FmIc0Z4 zm~%XJjn^3aPVj1^Ym=HkD`4;}rzWUO5FtJK*krs1Ynh#YDB3*y)vIfeOjeuxseAk0 z8I70c?ZJ-E9$W(MzAs_Oi2oaM>U1yy%Z9n4JH$P^CR^|D^YjLU@ z%8{Y~*t74i-ieVL((ir?k^xnY26%a7TTUB-f1t%zrT6$c5oiQ(tE$lH+U_r z(;UbC)jdps&dR-hFW5lg?pTQF!>Fo6fDw?R7YZoYu0sTGIH23{o<@F=hti}&~ zU8qTq;OfD$9zs$!20U6br*E<0#L8spOUKMmN5x~TML`yePtIO@OS4j7iXP_M@CKJI;tj`ML)kFZ$zYFW^JIetg})y$OHUnv3pFM@ZUw7qm! zse5f=^me{fDg~1|`AdTNV%2$^GdiDn=;Iar{W(w82Z-$4^T3n|OUNEb0x2{iRTUKO z*A)RR8Da`MWT2ajG~z~{2~zxM*biSBjyYzl|ES(1=c3=-Wr%#C{Rd4Wxljv86Cga1 z8R1Sxlpqx!<#V~TQE29W-pb6(EL@sNVrhxF&%_Y%&J?z+jSaBX8+}AaQE#~6))~Hb_Wi<&t zq}{r{MLYAS^0zq^UK{hz;#tgwWelfZvt*>S$BJ_=QGcBpry9E_`QdH$#pr0AY%?8Y zWe3`iydg>vBG!vtCsvhq7zjN${`ji#=3l!gDXnzL>f1vOv#YYRD{e|V*Ys6^M)`=~ znk|w?k8x=;h(4z0dqTH&le`emp?jJRua3E73CQ7eFDUp?d?wx&-56(qIAlD&B{EbL zWUJ5?;V+oo(WTIlV}ao})=%nusiGIN>+*NpG&MYb>=ZZUu})6Kx#c%qVOwhC>2`*F zn2*}WK6$AYV_Myc-`2&w;;}JTEVnz6ZMuB5Nwaxqezb-y1JVqdsllDj>1-iNDF)@6 z4OS#&u3dA9rlwrIYN8eYarfenrPraR8WGz6I#c>XIg_ofnRRcgt_eDloA+@zm62+W z=49yd8KI}Q6>PXRXq{ue`H-JFeCi@kwuFIJbQlQH^aCrwdBk^AUTnncY+Nt2vBW@Kx({ym!J zKYa7nKQ)Sac98jX9Zv~PP0ebN{gv5ban<{mwr{3hGh3)XwdFxb)~W8ctA=})xWbt1 zg!VgZXj8lDyPsw;N;p`-BQ{kw;~m?Q_|Y9#o60pTKF-c2Tu%ReUd&Veb#Wd4@f$IV zaddlVtIT^wgz5hEkQBCqWU#c4&?DhijpP}$+;~R;&dX5kH2of_Q`_aFJS$o;pp3 z#e{)Taqh|kJ&oKpE5qZgpW<$Di#bhjQgqVK#h%{u&G(8!nZoOI`MZh2!z>jm3VsFV zUMJ0}R)a?Nj3&pv{zFe~RC|7+e6ChejeFv+Gt&>L_+Bw;rO57Dwlj=I_g#vnAHa9@c<~?a@|4=)Fh|To z78mNaZla9*%V%Xi;lu_NNfZz}m}8Xv?IRee#Kf+11vj3ZzZAH-w>jM5dIV?ai)n$4 z;kGMQtZMt?hX#8el|{3ZTwloc8da5l*vk2L-|??cD=y!-AK;>7|If^0E?jG6l;*sT zW`^PM9UK1Tba0*y36i<+AZRv@_q;^$;?m3Q1BVkFI9j(R+hi1~962i>bx}@V?7=vyCgXlSn-9baY$wd^wd|MoQ&4blv%`)Z#)Qxd&);z9S8~~*iFvUK$at!! zwkn^`4&2qZ^1>^zNvw@Sw4txY=20ixP5q3IZ{OZTIS{Yw@Aukw!h|m*!a!6O1Zwe2 zl|O6fb}R?Hzo#06Ap9ho30(c8v3Tu|mLzlwd_zWbfG`-ul|Jbg@=y_urgON?oDfbevK4 zF5h%6&eU3})7yMJdWVMec;-^eoriZp(yQk6UtrU6uwN}}ZO6?6)}=J(O2oRfT5=sD4dP7*4I1y zqvX#N2`BEjRdVBFN}jOdwxb$V@;Sv35t~>J?U|e1&ClBS)I_2z!Re2=*^VG|?7z&K z26LQe^wHkm6-Y9-mwpr(Da;Snw?8_3N&X;M;#2u`-zD7V0#F_NHt>XIMK2QUMcrv_&z;$DvPd1WCGGq%FKkbE<17u79$aL1dIlVs)++4A?UforCDYM+ePT>^|ZZ|dV6C`@bX7Z%ewROJ^9}Xk7eA7j(!>Yd|owGN9ph# z$)w`&pmiawGl$m|w6%S=wzwNuskmnNqoa$G37<~pyTKiSy!N$^z0|554}YfHV$LCd zc5m;V3$m@xw|NvDjN~{uztI0&S7J~%!ugM^Na6Uc_$Lpf5>0!p)GM_s`(ITl$nBLO zn{*$_v<=mme*#DGIiQ`LPFh{h_GEb9K;{ahXvc@Tjdm4Ke1TNp=p-2OPMH{Mldk&r zT>FL__bC1CceNR=)R-A6y0!wH_z24{(#gileCvL^od3Dfe0*i9MCM!PPWz1Wp;>!_ z&D0v?m=0wW#cXD$5>?dXQ4Cb9)VDePvSqW+acak(LB?&4r*t3e^GiJcCr07i>><7| zN3)<@-qc;1^vxkmrQ1JdI<_ws-xbbhmRec+fK>A(>}x2Ogj@mQL_mCy%#Oe)6)G08 zYHHJ?V227mr>RB#KkF1`}0z7KcI%0u^-P#*$m0E?H~kT5&+WG{0>DD zBFZpuWuqvbKB*lE5&BS!2eOAKkx{+KpMcrb5{kkQtOVVEiOAtei@;DjREh*c* zmnJuy%QQ4a#|iV$J3T!;VOm!Q@-pRo!4*af14+0}-G-pokt<)1K&|Cva{+{Ua{;Up zHVzO~%(pXoR7Ri<0izM9)dcGf`nKoq_23fXA^`ggF^UX*(SqX=_U_ zO;AIFrPbK3BZmuu<-%y|TO!2)Z}1lAsMp{q3+s^u-0o-))lRJ{B2&ZCfb_c$;;Cq> z9ue^*DtcN>d}X*!t5{A0ulxAdku>mh(I@zb3w>zD`43v=W{J- z&`|FOpJb%FoE7;iHqjlW#Gh5H-kisd?SeKk$bSN`S)hE)vDgA)9tSwcEm!F0k4HO1%Wr{g}Sff0y=_ij1Z~+HK8p>4R5e})9jhi>W z16>h6MPrnz0<)DU0@#5;SpmI-wyGAo#4zer#)v%B$oTd1?qjIn*UGKKZ$hDl^vlJF z@6g)t&uiIPf0Vkb4J?bDNcz3Lu(-8}rF+vh&-4Myix=5k``b0*)XQ`{i#Ba5^H&ai z?e|pfp!Kf4Gu#LIoJD-RRSlDhW$n{Kc~yPmm@5wjJc<2$<7Asy`t*rtlT+ZT>j#c*HsoM1I|+eka}2Y)Bn?fWw~=?HBWOE^yn2?gBx)RJo&lYtc z#=8zN5MudM;Cm9xUcznW{_Jpj-`Lor^78V6=9M&<3Bqg^W*&?{lzsv}f*v*t1_nL` zvSw-NrfSn;Yo3a0h<2YFVZ3?srq6*%J*Xy=mn0l$96+b>t|;)(Y!?CMU}&2n1A!qZ zSrAG7*7S3^nD!$rYfx>oU%mR&(uTfTu3fhjh>EvSQDb@-0GYh|GG;>fP0Awi*@V4U zJkTVP1uSHZV}Q;EDOExAxraJQ5I_r%F~WiZnYATV>mHJa>G!nKJ^lTaAbk)>2H`CU zi%j;uIu8JPbF`X>t2vshf}mwWI#d>{%Wx=JLczTk01cTFwrNdk1eq4}Dbl0`ZL{k8 z_eiv`2!8>46mD93;k@BTB!pX8#%oi`T~;y)%dtLn#HZc28;O2Y?&iKk>a z0ANh1MnnAUZJex^YgyKrZ>i!{){IJZ>!o|HOi)t*Sn^!8ew0zf8=544eh;JBe$mtW zL(EdRIS$GCDJAX~JNnRy*ZRYOBs#{JPw~%OGoz3xe6ryPbpd`E({Uiz*G9Xq*QEuqyS%c~eAjcdNTNJEuP-J8+ zu;Tkr{sY;X)obA!>Qh+UVVK@6M%K}E;z8ZFAPsL?EVX}Kj9 zqWM}G`bR;5!_?VBii*z#7~v=+iUq3IMjR!RAPAX^5FjR;N(Ra&6=0tL1E2}R3Kc;N z4Zukz;W~o^aaC$4`v{kBL9-7V0aF45uuRG!*a5y@WryC=?SIk9GDn0Lx7aIsRT*EN z3Y1FXRsfcqR+)z@1(Kd;14^;NRnJ%wu^mFX({jv(jdgKA%D0QLY9Dzl{Jew z%F9{*6|$r{17c&ypwDn zjH!vo*Q=_EWj$OOZ~3#ky9%L(XbV%Y$HZ`%g3#YtC+(AwVVm@1Y~+u7oRjsB(eIH_ zQSeA?WGU~A3$5v=bk?od* zyxy<8TpCG9N1G9@^wC;hlJ3BO1jL5b8V~AyB%?$qY`N-=y)(3)KzwmY~Rk^J32emyg@T2 zd$!8%S?=PV(Y;>s)B62nU>D!y;Nb5~?>2Dp3?93CciekDL9W0L!_@u4n`x6@QBnd& zj_}lrQTVkB^0JyEw&p(=9k6@Te8ESVFHSb9{*p<_{t%T)hW{z^Vmv6ZI7MXM2YH%) z-X$uZu&~`=__+RYX$6FjJ&mqPeW88dGih%raFdDFKV+>nUSBl=2^xrHs}#4G7$fE+ zQ1=XVBwxYx+f(Kod@Y{u!u)#CEnn5=Z_yV(qo@E=j=gygWP?(9A|pNlK7~ zn);R!&4#8L^70@KDyXW$k~wq6W&8-B9DuN5Bd3B^*d!!=hP2w6nN7X^zC?{=e`QA5 z+lfE2AZtJ{sSy(pl|x{7F*P+696l0rub=?-+jbZ!ItHYgBoUh284u|1Uc2?;^LUr zPWbL3O^ZweIu`r9mL|_YaX<|M%CKp98LTENdkg28)C_@vv>=k|4yRqUv8g6m{$qAo zTrJcXNfns=I7o;ZNIUTPOl@o)WM+=raL;0(e-OG|6QXn}qau2hm&XYklbjqO zR1E8avx1Mif`1DE-In5en4M@w3JCX=n^XIIaf@~4SyKns zU++%Q;IIgA1DhTOh}8)<=f}Ib|3~Q`K_aegrxczkb~9Px^z?(p#jC^e;c)E&6^vxb z=

K7CwW1FJv=dKqKL?K&PUjg2V*^7;z~n3gMiu_;6vH!g+zj9*l(K<>g(ox7YYs zzMFx8T)u6NYNB@znGE0t#bxyq zI10$!jg5^WSH7xeQ2kn3oNh@!7l@b-A9^NCC!Iwo#2X(!aUy*=I?ki)K9(EvaJYxn zn^3#}rAbES98#wSaTmmc_wV1!+7>?GRP+v93c9k3#eQ8y=fcqg1>OF1JqvX=r$n?Q z-$~DV-Px|RDnhHoo=y4bSi?gXQN5b5^ulbcpW(5Vy24eBO2zM>6IDoj9>`(rUlE_>Csl4DcA7_$+sfqRma7 z(}(-ptDk<^9rE;P{PjM@J$n#@wvl!LZ-4py2kU*LGnpoC{6jE>5_Lfn&wj5< zB8fmt*;FzGyU%(@pi2Z9CDKVoy@v7-qhG7(+s@*5f>tK;j#g;CURvklcYv*AoczrD zPzOc*Vcw-iUwHIQCWhR=g$4nCLZ^mhIk`cpCAcP>JveTWX0z`0xj3{PPKrPW|NF~nR2>wy z|NG0WK3L2D_m>lFWR6i3f@V%Z>;VLXKk3_KgA#1m_U~VV#0<7;|I%b#*e^{rf=qGbPQVuPndp#j!sQ4OVg^qof&T+*+^tYzof+?r zLDcw(gc|i>N{gN;5D>_QhQjfR7|Q~Bo|u}_EEpIVASy5TJu=QfU>`&v2nFistpbf= z6uCr)Mm<1@Xb=#=h7K8pQ^nFkUmz(paOL(QCqoIw7qWwy`3tP+?*OwV`2`$J(fN6H zkCv6sm2lDV;K6lE1Q1uHY7_h3m|P4GCB#6L&@klY_$OoN5ah$xW*c4 zYHL(`fZ4p}o(d?|55me1Y0NzUh7GUnObD|Z;7y{(c( zPS0nm9KZ^#Fx)^OL*Sd7%nholII0$%k94p|L_I}58b)#7On%%KDdnkuC=l5NMtq3` z9H6%)(3@VF$|eqP45roE3&}ndNF?b1EUhFc<&U=|`2=pGw|LveHr?^l;ZL5ByL!{c zjijzYZ7q0X%rL*0NC=!^^;}EHG#DMYfPJd=61uJ#pm-59kH z2rt^tH8t*IedJK4?Iy>u^(z885y_)vl}doR3#o0rmR!0#rd24gv<8w;3Hr4U=q~yTzqWWUVg^Pg5?z#8!Ph} zUUIjv94LoGY&#h+A`BO26@YIT<$k7LM8INT9%!I3K*R{B1PH)vCr%g~>c%BZTzkO# zg&Oirq$UIrL3WlHYKQ!^u%yHhhNQ6lQ3hd44^S&UK)rKrWKhs4!dCz~f!~4{k4`ml z^49^?kEM%BD`+suAjJ@xjhm^_oO zYMlZSQsRGYs6AAXuLHzUBNmCLHnKw+_cTyAIY2^u{fK3$ln+3mTARwjM54NnOG=mT zp9VJlDHI5>0zRpDRzZQz&CTr{OnqU`sKeL?i`2Dj;4oDpoQBN6dAt!q0Eo$P3H<{D zTkvPSxOg^i-W)uGc(_T%W3IzWO@bBTOS^o~oCvPi-XbLR9y$ zL1*bJ_aqqz2h_%!{O*;!u zJw2e}0(^>a&VhN@d;{5Z0i5f`mf$ekp)3u-UzSJ)DVJ<)gdic0tqQ(pN^m!9m?2D% zmZ%af!N|c8uT;7oWo&ELw~h{tlIymJ#fb^ausuqIEI+_-3_{L*IN$BkM}v|ecwSsw zoB-2{(R=goShY%%KZmijzS!ijnT|3_-6V%V`Ij zqChjs>;N44?CfkfumV+x*tvrm*TuO*-uV54KW;--4V)_zYx#fBok1*+hhi*dWnf0_ zh!2$SDhT@p%C+m)y=-1$xHL7`(#&j4aJ?wGzkm<=OzF@-=qEGub!==;c$WRq%Wc1a z_+v`%7K!L+{++XF@R{lr2kPGkRy|1&usjygPoU~xmA0ii>jWgj3B@xuHkNPS;X8Cp z;!gL8gay)h6xzX#D-zBm%WVzxKsMK^jhdpfv$J|n%Y!h82BYtSduk=s0nQXK-g^`O z29D8j1x9LWSxr*FX5j9i9dTtOP;G?@o@O^!Jv&z8!Dxn<{)>QcuTC`fAzBWFlC8LA zkW{B%-x}y&Zsv7IT2)iGwdip|7%^VLs>4YoNqLTujxIJJHOt2LOPW-TItGVka+kh{ zc4I!y6EvRFFuftIDD)a(2c0q+_Vg+BL`(f}?(bBddKgAFS-DU2ynTaSquL?r>cAgJ zppmLQL5~5cU1A&c#_D~?F?Ip3_kAc#aU=B&4W&wic(V}F7NH`DiRld;nn^)TilTlH zDGHJ^Snl-o_uujNuW@WADY%aGls&nHfHuQ$hm09n1eJ%eMnJE-6TbT z#Nc2kGETNk;e<3cHSOo(N~|75P(;3SP^}+7By#2(C%zp}YoZR6~3F0J8Rw%nfK&SP?BvY#+=4^9<>DhQ=+e&;Ob00o!rQx=2ObT)=m; zI_W}TQK)TO{vi1e_Bl|cvI#+*t=owkGZ=O&&HdUR3w}d38A=ccR6i;yDG4fb8q!2P z6(;Tej0BgX?Cul=&7-ZLZr`ROV=8K2(latta2qCCjIf-6PJ!!kBR1PXXAPQ7o2X)u z`qne}zkV%>ZeEs%unW{qKx8<9p0{=rdY`}m)eTM!2fisSEpnUXL&oW8vmEH)>}-Kt zawGfw`}b3X)Nosp**}!YkXohk!QqE-muA1vEhya3y9oUsE`aU(4AZ}41qTWS#?tAdVL?GJoW5mU0}&MhLa831b^uKvgoj9M zBf5&WL~SQ)+x;N!lum~DgcaHf)Dq1{$Migb007(&#=sjcqL3M&l%`j%7^cScB5c$f z-ShX~Qe0e2jADd^g<(@(3&geMPfkIBZpq>X7IzwwYbTr`WocRxOgI%w-DaPGVUJvn zOH_1sh>{#E(nzwh@BKHRQ%D#{SAL@W>Vo%2T%X~vCKf7N0j^G3XuwmtV5(=u*9V1w zq}W&=wC5cL>X>7fz;+XaO*WSu(&_^T4iLR2^o&VTjL&r3Y4EX7b_ZSw``-*L zKT;JTpFo%?LQ51%pW1u*5l%ni#^d4VU%T=;kA2Bbc?x@$IEXtu0$K_^L3u5$r|rh> zG(LjnAJuUQ;=1{WyN~zaK@MTzr-4#`??W977G-}m-7OTFo-RRW=-0qNx{B{24=A&Q z%LimAF?j{sl(?5upb&Kg!x1DofEzFc#8??u4igxJQE&l}A}ZuJExG&f4@hoPeUw*P zItESM9XoeIJhK1yZ%D~Vo-cCX+OmtE^sr?X_x!`Da9Eh=9XPCob{Y(;UnD1=k&*Ec zs;xk{grt3rOPGtk=!|X3l`+gG3-RS?T zm4XpsaV-K|6(@~m#O2tmEH&Ip;m7S?bnZw7;9zQHMfe#oz0r9dIp$irJogQ1M+{&- z0*suTn2<@-glcX9A9q{Y13=f^1XrzA#x?{GKp?5m6-Lvb^1}!Cz;XbjS!sFd#AElJgnUSR zZ-*?x;Lwn%g+-+g6*ao|oiTjPXv^X>8q73O`Syvy81Ql zOg$OR2BbsBfDz}!vp1*KpT*O_L<{C0-$G0p$SbntXlbKB98_)3OB>lg+$*~6=pr^C^1NPp(ey_M~pa>X5~(whBfwG$T^zVc%a1&jN72!ZbooE zuct>IW&Zl@>}%1zBV#pybbvUM0S_!!h#4Pju3_{1z07kTNKyDUCe{`d_O|K9+ulo9YFD8WK{HWc}Q#L*a7))IsX-Fj?@?Jzt*VUDU!=FFK55Kuywhi}vy ztWku~#)gIp7;y$N1y2=(jr4$N4Xp6@V&^UBhvbJy@sTF@`SV0*rLnbD=kDIb ztg;77a9m6b?DcIj2s!|AFoB%n*>$i9-1AhE^0FXfef?^L*#`XWUQq;K zF{eQ~6nOx_Y4N4d13&JuKy;u&GLy-DIeXNxkEv z#>?DWdlj0t{1X+SEa?Fp!wvRJO*%Y`)Y3!alU3^}Vr&0@Akze!KcF)N8dd$X;Njsx zhs-d%-`K$5IgFHegnL(;p}+`>z?t&AM@U;Au}arrQ>-?*aSmnu#*D)A=ikz|^(8(C z2uMgt8B`nn*YlPdmf3GHzJ>|KUVu&lV#U(vdg4#LZ|YX#yhGuEQO%(y~Ih8#sy} z&a85s(H+BSKqlM-h6Vu51(bQXpy)uL?#Zo}df(k01O7=|(+A6-nH&;<2yYHBmGra} z%2whMfN42Yy(x$tWZ(=->1~>jbr)S5JbV$GmcgRA5^N2mX?eEab|KI}e&sVLNhk>4 zn=lOJ8UV-vJZpxkQb1c3NGq;&E`I(IF1Z7E6s3GQ8vtYr{6G&KYSq87NuZne80Q8T zlHF3GGxT+tq@~NT$h)%vyb@Fo^UFEMTu^5Yq9aEk*3Cj^3ptl}BGZ|inOR$H6*~UC zXnMyq5Se|z!+>uQY9#LGBn1CRw1mR{)YQ}vQ(FP1dFdZ`#*=e%2f*n;oe`!xmb>)m zzfI!$A2>{PZK#P0=msg&D0py@;?55M#zLbkZSEp2*=FqN?YnoMadHwt#0Czw4vi|@ zB!prCdI1?Lg!MWaqI`!;Zy?=7=hrVk|0pg_xa;JiK^&W&eijWI>mY2gn{_=&ZFG2YfG*dYQ#9M1g~XAjhz4ml+6#$oK%uy|1b|(}Cr3hb*2JxO6DX zLGy%j3k}Yug;*s3Yp>Pi8^FtmY7LSJl<%7H@-&1E4Jaa}kn@H1G8t_{EQLh6(&^J{ z5p`=6!_i=xqAIHX_n=SG{^wfc@bJ00U}2HiqLr=v3t1M@ zx;QXnbq>eykdvnW-$^4{nh0QQw<#A;^$^x2sDFHF6SPdnV}E%g1D~9qw}NL5cEXN5 zdp_Z*-JD7`#aF;gH9Z5vX?S^HBiTq&&>tYY7=rVF)EJB8o?tMDgcOL))S*w(YBmDG z34#R019%X>Vc}!^kkCW`fNb2f$rPPp8$lcq2h<7w3uOe`Les^HxRysAe*$c~zwkfF zCtfX7D;EG8+57`t@h%`>da#zUUD3$RrdLBuLCa1M2v0vyPiWf@+hlvAxq=ou8HNLb zuAru7g~ltvtqygAKuXBO6vEq!fiDh@B02nkq@}+JXr&z^VHN!f*MX=T_a8hk!`l*H z6s#?2JKzaib8z?xsGm~Z{T-sDUyB?;U=bts-*`w$7BGPiM<*Wz>;m^kOo7=0z&YjO zQrM*KjPd~zjbGt@6CNH;pl_^2tViZUB3y?vf?W3wnmIrmNxK_lYZ%c|;fu%O=C?U} zS2aS!4@*MXb=dF|2*$O5Gzt9+-Ep`}Aqh9_avMIBzXu4nY~qO(;6jA*9PM@$ouNv! zFK#1zP$1`Sj%peFv?D06!DHT70ZJh`OxOz?JUsVJq~PjA|00XipQ;CxQXI=wVav z6nPO)k}uG`4fX)H>OJl#ga|4+G@&YPQa838_SXG>ihI*|s@FGsbg5mbkWwj>REkAp z%20+VMT&})OeND2GKGw7N|Q90g%Cn4kup;mvPw$Hn0ZLX%(HVn?ce!-&WrOoZ%?n< z_R9JW&vW0`eO=dmKaQ9R36}j6zKvum3n=vnx)idv(MS#L;uGdQlG%+y1}lhB!Hb#C zw~luTN|u68hIcC!B==>IgXwVY{%f1~iU4z?1kSjyBwgz_byHg?ck`)Lp#J zJ8)s}`LYA?tb>5Xm_SiQm~cokzU<0m<&pvJj0u3Cl{omVs{lulO?5}u22LW6ZScl0 z>)pp%ZZ(`9T0?>FK}!X)S2~2?gjW7 zEWuSkadx29$|G&W&;U)`0$uZ1rvmo9ahd<-;9&2Ja( z^6D8$kHY`crtBP!J8L8=KNE_)tnvF5O^P##&p2#{exx z*jH;YVl^AvMtZ?SgUQFO`>(zrhLw@R^mHvG*%Pifo^&-I)k_2gKnW2uOFFu{6JC2D zaJIvHtljKIo-L?Sd^ubUC1?x61IM(l(=0qPTI8yk`M`W#-@A7&J5d{c@1auHAx@Lj z){ece4DAP;rVc)3W0MB#aON{a>U$p;jO39}CQKI+#lG4N!SvA8tFJQ#zRtiOy#r4; za1zX5J47wrWGO3MbO&DQ-QdW@2QZ2*vahjt&@cjPqH8J zb+UunD1)6i45RJffQm(tspKmC1?((vQII=;f*If&DM29PQ3OnCnRvEr5!|zf^n9p0 z|11|25!r>)9*`S^EU|{%2kT(BJBl5f(sUIO1eN3WsIkr5+ztzKu7njY298cSP@jy^ z_;D;87(reJZV^=|!25M1J|qI92?sPGKYIMQg26a7;e-Pq8yjbH0y@-?@1h$Dbd#Iq zuJ%8ls{|Xyjt&Xb4cABb_F(VnM*UI)Z8 z@na~nplE6oIK9?$aos^EBQ|oa<{db@g3B9-*bY}updn%swq+?mEnKs>C{*S;xr4fn zngK|-LPD#7%@9?IMIIxAjS&uUI86`q4Gan866kpb3xVbV%qe=E4tgCtF5aFFvDT0z z=RDZNCnk2XH)&V~D1o$OV22Zz0KWm842T+t?b$GkAY7@RLy50mcqitd*Lvg(;P0SG zcsNN21PZN&11R3_A0_C-OmNzxF!_f9qMtO&Q^7mO4SVVxkJu$DrPm27O} z#gfB-@Fs=2q%5m_At?wB8MzHenRW%D0jRa7$FHz(furg!!0XuW3O>psZU0ojcamMu zPr$ziw{KuzkQHG9)B)uIXV{#qhM_TLPTy|n2yy5EAJI%yc;{mcHh?U{NgR55hijY4 zYe2$j0QLvf;!vpMeF;U$a*I=4HO0l=i02fLLd0DT!wAYb5Wg|Ola}Sf!r;1$u7y<> zoiFv@66nRo%EI#fwH4ruihk4)IM#6rQ1VD2UL1)O1IsyDamuMVC=j4h0+;b^(PwQ8 z4YvX?GJ2{Z(e-w>0Id;E0{SDME4_t64{`dXRjmkq7~Ay4%-p=Azn?VbqSky0s~_k{ z0Vg;haezV*D*rN+MuBZ=Rk5)t^F;A{g*S%Hfk~3@8H`mpwJI?<6Q>>lx*E+=T_)Q^=6#yce8J(Zv{!06$jeP<8~k7`s49te{EdnoPz@b-Xn za555)G2qoM^lKvCWAa)o8vTwteg!_D!44CLKUT zhCl8t=ooYs9yxuQ12bF@&<>tFxfXgd69uJ6+^bjGVRl97=?`GJ6YO&AuUiWeiIlEj zug+o?i9}S;NRBjCMk!=wKrPbc1Of_<6*^BNJ`<()8CZd+jlg@Rz>shn+{<$w!l@yO zkDokwp+D<6)4SU=`i|>5s(zt+PW7}Ear;5>zGjTc%OZyP9|&tGElt;6u4O5_fczO~ z!LIv?0~0{C_=4*K(dP2|rLybLV`zcq459ynrFBfg!^1Ht3Mcs% ziMx(m64B50hVW!tFuoBfhJ-FmEN}LlwFc0ds*V~a78gQ~tzCN%1s@QJcH>}s`#Zc0 z7$kOLej9YXWcfC?aG*=H2)u{OgS6#A_aO~;n*HG{T_2;YfiD59P#_-$DTMS{iWxni zmEg%Khq5<~Ob-WXd$iodDo`aP@e}qLH9RuHn7+K=&fdeM0}WO1mHs*+;8Cq@p|Swz zk#VuYXVblLVl@c*dJqzi5}|>CY#^*6c_RRB-xSrq3V@boC~ zgc^za9?xW}o1vtXR4aa0fJsgHBAn4=VQKjizWx6( zAix4;h)yUa900zESsw0T-t(%+b3h2%H{~eAGe8OwEFn@MC``331(6e#{SOU>)o8B; zS(Hx1b%ni4rUO594@Cq2UMM@KY$oRCkf_ReYoZnuD*>|V&=&w}7fGx5Z91|!G5^J1 z7|XPeXW9JJ+gw$Tk^W>L3cxrQoMjJ4vO#r^CaZYjFCG*xtXtReuir*OcRDa<4T({kOu>d zQjkau=Ek&P0GlZm9yv7-y?3VlSOQMba%YN=B^mbTDR3+F;CB(9ASMPHdd)k*x1ZSb z;x8w}CI9SqISgVjuv3RBKYgP5Z*hiQ3pEafK)cPk-!e>(k!}{qZx}qqiaI%2d%*ZV$tt_iyOO@u$+i`W#sxzi;!uO zrIhd(j^=D^sH;o&oWDlw=xD(sf)jc5C>D(I+Herb;LD%$Fv5~o6DDP~5iC$YSS!R$ z7#q)KTv`dgHF~V=Fbt*KPzW+e335XY`l$h3-h&W;*Aee*RbbT`a&<7qr}j(0RgXwi zj93=*3N?qosK9}1O&jDD2``5oIt85w^omh>X6ETWRwBC0JNzFiexi!P;;5(bQP0E3 z?kwcq-Pq|MxE_Xt=vrP>P@73)h*p@jJsgSHZq-p6TIVUM?A@`y>zJ8d#P78WIg>*7 z(r3tt1jz91V3tP0$*?}x>E_(ubPiZ~Nq7Fh^K1-7Gk5q0fY@_P`vi#W6^ zcr6Bqp>VIcxd-10h2g|spX-}Ggb-{^kD3c`y^4~lQ^=E($?u6J_L-M(&Z_?}BP}6` zlXE~^NLW@sD^J^D0ErJ-C~y*Q>Mxme6>VlosjP-E@M$wL3=o#OArBK06?+35-BB>} z(2{k2^rHF^mUwHw@sPfbIa-Q#16@9EKpA(sTyx zKB!@cdo7~~tUr7f2jO5~ctq(!&4m6<#$YkTFNmsufW$ft`ux!-9?S7lv|=>OnfY3> zV8=RPtpEc|wZsQ__Fuh+rJ9`DVhA4K5FriF8K((p4{;_D`+)FnH`>VqguU#|Bbbo2 zS&=w>*%4&fd@`I>A+CHVF_@0zG&X*!Y<66j2pm zEZQ?9GUzW6z-)*RwVZ|ZsWL@?*@u|_yfc-jLBb>sr-2435S50sdL009w(03b;HyA^5Ifm)-ow97>=E_x-zR2J=)M_n2+p>H zXU-750ik6DfH#=clqPfVW*{ZD`x-++fWYU2R3+3XeYwsXgP$7|pdx`9_5-Z~57hNV zDRJU4P*;G;2T7V0VB&02GYTjP`_2#Mg)dX_E2^r>0Z0)ZFJeT}<;jVO<;LjN#k9a)fmv zKAr$1YLL2`TNF1Pr3&zP1)yKFYjlX3ipo;NSS+sZ_hXb{oB$**ZfFPx@N4VnfP*JC zV;EP%F~nw~yc=)ShzwjR;Z~(ZK5qQEj4OmyGrW!J!_& z>Wv*ioO*~3u;dCHeJFvjJxVZL5qD597PaZWjlbwf z8VZv6P_YwCfU0U5X+up@b2kzm@csd(7I^f5w+xexxkK#+Tp)fq0yimWmAC^kry8gS z@M7kBM6n{+4EQYlNy%!7x>H?#1AVuAq}NFzf#Q306l8g&XzPihjR1(*<=#a9!50cO z;sPw2^JVem!MR@nEs-^k1{(tiy`+B*ffZxmGcm~8bQZ{pJRDRO7$XpSY91T&a^q`% z(tr%+2I@BktN5CK!S^90Z-b8;&OsD%=iqXHA6gL>57PM# zV;LACGE+=&5^0ri&@ekIEKHeXN!5a#CEaMC=s*hXhff+FT*U(uJVRg=h)Dx`+XSTi zGgQCBU}PUfMd9eX?a1G!xs-8LI9zNCs62VxA^b+vS$Iw1x!@%r*pJtfSqW`?ve(80 zz$&28DHARQ_?hSHLjMak3;v`HMCUX}S=b#2LxtlVRPJw;pDCF`1(*nfH<4TLhX9$^ z9-tDyJFkU4k4r_vRyKmP7@{m5_YLf{#rYwxXxXwmgNp&6Kifg3>$t%Q*&FuJeIv$& zNFlAiL*0#I76hbA13*}8g$9Fyi4z1QuzHxZ?N+Gr?btb;U0rLrxENUZ@Ee08@r43_6CZyAgP`!5p+Mio^RgrM9QP09*m!7BUxB&Kr>xuG0nVD>Tv2Z{13^nj zfExvD88YJ`lzgD~+mU;*&2?o@?hITLJw#Qeu)pmY6vMmXlm;bW-@gS zJp{Mn;Z| zVjT(!3ts~LPa9oICJRjGpaj>v=K!(=DjU^-2R3L7@H8N^Svp)>dO6 za32uo_MZ`{HemoZK8YI$h21dmY9lnd%}BOr4OZIYfi6o-D>=G($%CjU!>)9;zfkU( zmjtl|RZE~n;2mHuvOkd}pe{WB)UYUwb>q^P`lb%>03xU>ZD}#;M{>$Tf5|m3egyjb z{QRiUas)Z7iNWTA6k**-6ir}n2nY^V`UJ3iy1Oj1ndZ3vRDt9m_P{CtM5)u<(c7t4q?ipof=sfS>dO`OHr8UyI8C{CZ6bFY< z^;w1FWWKdp&CE#m*!4939RYE8SE$y&>U2O%BPWce*?GBvo0!=~4nnPJY z)O%{m1tm++0P-cIA9!2lO5~_E+ZOBQ7o$r)f4&dyAQROe{{V=LGrMXqR-5$IV6+I2 zfMNn{y8^DPtgK|3D1uQOs&H>&&>ajeyf#jQ4UZUQU;+_lCBLHK^Vt9&CC+k8C|%*hgJ z?TVp2*JuT$rKQBHr^D8`Ny3Z;#ow7T9K+~d0sVqf3x`FJVMrt`fY>0f8B!R=pPxr@ zR|%{eWRFUTGyuU1Qr6tO6;?mI?@5gwQ-DgKq0RUKk~(06NB_v90C9LKiWcWZN!3Xj zF`mZFn9y}^eP?i6A&`C7y{WTNVX;c8YsUm5nv zWkoFuo4x;^mMyPUeE*IeZpqFQ7gCDlb@oW}e#}0dQ#w?AfD}$F~AfSy^4(R8Fvv z#)%WEFzFpT_L2Ig1*UP7#bnYp)Q`Z5Q7RcsMXX>r=?B?qU7mV;vLhSyq8B`0a6bjz zPi>#nfpq~e8mA_#Bsv1g3&e=kErGL#6mT18z-0#mZfEy$-O@nOQry)YP!xb>O4C{^ z$Za{F`x?<2@CVof-B`y^g#}(%6#(Q5pXuacmUEBY$=d0qFxNWVigqhV?{=`I>*e6DRh}At7MM@$qx{s|tR+Bo37(W^S;%yBp_yVQ_Y-g(UtG>1l4``&Rqiqn{J08xKZBx<2!w z?&Cf*H+b4DG+?#So)c3Q)2ks^y$)SZ+6IN_&f90B_CE% z_Qy0y`U zY{h(+mv5)w(}CmT7v#!042o+*F-Fdi36n5i_${=BYu)C} zr-lRpW450(kPk%@3<=YU#(kM40x-&b7t6978# zZyho{g=k7nf(2NgpFU$qiQYyAbzeN_Z7LpFq@l7#jM4+cv+=F@vn2fCD4Uhw%6;4e5=68ggwy$avo@&EVVC7z43 zEl@W;Ait1F6-;DiJS1!rNZA^&oq$B3Oz1GJkso8nY2IYi6lt=8L2W59$AL{N2b>Sv zXUzrDW<~MkeVJ%%AXC^unGB>i?AX2A4;5+5x85(kj$|ejVl5M?!El@BFna7wll^MQ zQ$UF*Qq8kUxV0BNS^`j{K1d+7wf>gn#zAf1`GSo9c%qSaGJN9S_G zE42!8eGqCE=6Vz*Kv9+5-_j?U=Ou-Mknlk=WaPI$JvsRzz&h3(YTwTFUbt(hz052u zm+*ub{f>Yg#zV%F=v_hKn_=;FI|^QmI5-mp&Sp2x0YKvrWQcobQuxsy|G46xy8UnQ zsK~U2PZ83H;Q`AV?VTclQYnz-c0l?CAR+2g#ii{cA}2N<&0buX#WFsyI5&zNbG||m z4QK_1Rj}Y-^RS^{!Ic5*3=a<*x4pX&T4_cG(Y@$@{^CU^peRZDq^|~q5(UVO7JB%lr5Zl$0otNX|5C!qtVZ z8!TG%q!3Y+40ACJmQhq}iFtw!t<$a^Xz>DOixBFpUnQ+v`i7vn{_T8Y_up!Wbno6(Hw<$tDL% z6TGs3YUJ2xI4PJ21`Cj%IO{Vm3Y#_={(|sG8g)>S-A9;QIuSO9Y1qRbb_7uNUTIZA zC$Aa^BT|Wh6(RllP~$Ml<53Sh^I!VL;kpzQA?h_CdCFV8+ zlI7`ZRA1<2-2qSmBM3KO2Sx+5qN9HoWI!y7m!LFk@&@)1nd7mhf*OQT%4{p?Q|hyJ zu&{XktxDs+NG*DfY1^UIHE~DQkUnRqC}a#Is;lD)5+K!0xr@oJKxd0eB1eDElDRHX zuL~eM$u7k30ve4LI0Z>YFmiDQrR-i9hv5C{ZG0n&a!gxSS6Yvb;DBDDE$@ENjRGpo z;&OAtK{tuxgG3!kG)D*4;b;X9o_A2?f+AL;D-W&4Z&chtCkNJo(mV4`O&CGUD5co||Ymlz;qWJ+^|G{`r zF)ZLk*%UKj6qbwU2GKP#gBH58sb@$#>nl5a9C3dMoGS03TOG%aB>FH!76zX;yvcBu z{5T&1`WGq;*S!9J*}AHa(}2l1>zsi2rPGEUD>+kld)i#c?sezZFrs}*e8o`M|EurE zM%&sStO-(~>H_;ZjWe{{2Dt78oEPBInXi;Mm;C&B&Jyo8$M({)f*&Ox@RO9w5=xV^ zeMJl5w%$eI{HU2R-|6sb-2dXyo^sjH%=;#suf_MyIXA77QBpI{JSQj?rUbAVhj*TK8kN~RQj{CF4dTk0Gg<Bq`n0jWvQ@FosoIRDe-M-a0Wtnq%dn_)XKmaoqufm1$ zsLRR3dO5LWg7q3NtI;j`{hJo=NyY}?fEB!K5(P2mp{~`R1#CQdTl6 zuQiQ@FOL=!5Ibe^TK2}938%MOCNWK4rZ(K8Ztz|9p=BAee7o7iT^35+y3@_N*c4h>UBX3VFkre zs=3He87)(w%|^~BcSV4)ONe5ePT!UhOcqzHf1I@h>m*<|PciLf`epiPvvHQX%5zQ8 z_Wkl2J!gC(+qoUOt8Fsw7V*<1eM&yBK3a7w;pyD^BO`BQSSa7mvWy3(p6p{DA3R*O z&*b*1h=28TCgVlrrNbi4gJtA78a;OvG~DZnUQs4>u1K&zx0>JL18bHVGw>G_jM9S1 z=3|bIuf&fTq+lnKiFI%@l1lCJzQ6rN-&n4y5N->MEL2TVV?}-X)r1^{eG&fnTn6&) z^8H_o6+^aRvIDpu+0s%_ir}8(M`@Y$a59=gX7n!bDIj$886lJxDK!lA55@6#qIvTp zbl5Kzl9l#B2518&=m4aS_q>J=2&&QIrE3H%Vd)0Ju>-jY$_KC?F@Yq06Z8{WLfled z4UHwaFtcMB>37<`bLX?SBUZXB#O)7`R;J`!>Y1)p@AYbGYCvu;nXukOU@@FN8e!sx z)rbKMv|>FN$?)< zToG-ye)va^qNH9z$_bV3z7gkx2%6bpQnM8A8Z$nA6Olxbn#ryv)f0B%Dj%-*J5P;$ zzw13JtQynCvq8$q?|N6@vrN8uzI$dpq6c&m)`6%Y1O&K9tpOAwaNT@kGeenB%WrpNLLZ+f>=*J98*jxpY`W(_yAsWaf2b_1Jsw zQrh;!osmx-IkNRww>K-;wso!hM=9M|I$!@^p5>>ToCcB`DRRuqwnb+ZvMz}EW;MuW{advF0HslxywG$`I;LXGVLYV*Q`N=FZqsNojYp^k~ zR^X?s0>{FWCj10L&rsjQ!4(Va%XYW)9hQg+jrO}Lqh!7}F?tuw^}r3)DY~KxGrx8K zT0CpexVwCcOp7eY%3>m-p>Y{Yb%OddK_}|*)Yy=VZNzBJfl&OU))I_W8U1;lIZ+vXR(6ALfhA6 z>OG3NKecC^Lj6y4s2-da6c@c5=TkKLsMX=H(doGk`^67mZlCyLo$t0ccXoBCuntdU zi}JN*)#9?jOfPFk%G22HRW%9rOAg2_Y3;4PbK%XxL+WY45bEb@#*QcV4F)a7GFOe> zzI5r0x!Aen>{HY7y*U+~AzOCf>H_cWG;3Pe$`uqm>@Cy1rtW>j>^ecI?znH+VqUEZ zV_ec2!PM>&8#~7SH!_g}iT%#`&SL@m2cCMxIjxNiM&~hr219rVjXp@^cG(LjbpJ%+4q4eSG>ifb zF=h9qTiA7lNCsltsgu-7A`Z^jy4Xi%vi&sscBLELE*EXabeiZ zmOP1prF{Vb{!^x0u&68FGV*7bW!5Xm&u?GxIdyq!nj!t0#HCMFk8*0v+OkAvPo6u? z{M;qGW%sLz+^n}_5%&);{@hKXundhzI=_nkYTPiE@9*wDCrFWd!}ZC-qd7NC&RSvO zpCcpEB-`fv*?%(T-6gS%DTmn@!!2_rp=%4arqef`Ewp7meDg9Tip!oQvdmxe#N3Tn zO{`-pBT8CtwPa5J>bSIFu>5deMf5s}y`kz;BYCDgMWHh$jRQ^No-Dt{nDNy~zB&;? zE|#^{6yFskf0SCJ)Sr~n+FM$hHBa20&rNmP9`a2-+h*n9h=+=Ru#;+Fc|sld3V9GZFEVMUP{ZlO)VBZEf*ymCPqRi*|dUNt0I~# zniEB94Bpq!-}m<=Yx>9EdAw{w)a+pl$H}i#(pxqpTd^@5G!{MDbT1!`K7OXnlhWig z(N!p-r}$yHN?M-gXte)nKy&)b^Irh8eZuwTXdm)C`Zlj0&N_)vlzxt_e*bd5x z(4ne&Q-A3c$6;R=@z{WQt)^9R=fd6=tu@s#TgVpSNU6LTjgj$d#vXhw`b*|X_RTP2 z84ip-_Mw~Rd=zR5UCTs_!ruBdseE6HNWQ5wlg30aXfiP zwYoD^wZ_)2|J{*VeK!m9l%AVDHv>0|Qt3Y(YrVqdihq1kJbBV#nU>h=@?$8`BHUGx zhHYC)ALTY$xi`@lHC+$he%N#B#1i(6t!1a3oh;k`V}v!rohZfSse`nnj~iN~!hd|@ zXZP3hE2YJLc;H`ZrW!}fzSf-fmQgq%6m;&@pTVyd3!A5TFJ_+mYSq-+&@e4BtjDx< zOK)x4K}Q+@+WBuk-Z;nUcK2p&WgD`tI~%QVcP>Mt*fYZ`u~=+cWk%VC!h`xsDav%4NGpQ?D$GQy0Ki{maPByUUYvfL^dK#Ud9rrbV z|5U9(Qb=%K;n!FDF7s5rG4ijKcUgbU-_6IeK>uF^0%iI>*wDO=#G%Mvf5wyU*wE-Q z+4(l~I7?xd*v0*y-kD|Y`|~P}=CI5yBUorC?{(p5Zj_LC=Wu;*2-9r+GL`L7*)jHL z{h7P!F>4@}cfWoyY9_lcdq?b_Jy)}SxIe7CE^(OhX>-H%M*65Mp;?;_gggB*qT9up ze~@8|Zt*RpUG$joX{tBnOD)JfWs=sa-nGWv*>ufPIsUe^$sN>w!_OPG{A1i$VV+*6 zfxvy?y{cM`tEYgR()pHVM<9-j%6}#n)7L#N{ScEs`N!U}(mKYuE%wi|{9Ny*M=GG0 zVjGY1C(8J5_nD^@G%9l&E-X5v*Z&Tl%BCe&nM#0YX|= zpZLgH1zmeH>K|*R^^>tIGi!NfR+-_heWu*LNOSMw0t8o3%dUo2bgB2$a2kpnbjkT= z-q|1BUSIn^b8fEqB-o+uDrFQYa`AnkzVMmZmhdA-Zg$w!2z$oA9PAIjlVTWMp!L>@ zH+orrEyu127W=b@wS=~paBv!Aer6@RzFqfwzt!SRN6&wswA%_wKIynPt+p^fu=Io_ zjsB-^t$6t3WzqJr#w$>H?BjKa%Guh|v4P_LQ+e~%zUaEE?VF3sqF!w1uh46I@LD&* z?vO;#y;0tUDUD47b2AJ1tob*K+}u!0Y|`LI-ZbyhHA%k^he8xJZL$VGt3rhCHQzUE zGST;bex{2)ww=ur4;}Vs^Qv?2Uuk;yM#ZU8ZnyT$H3PCZC)hRqNT4}5ZuVj;)?8DC zrSE$dq)FG?pVeny%HQ44n91|*J9WdFmipVhD+hwoyYY{Mm^8&{KFg}WKfCNyVUUEsfe|KQf~GO|tEy^J!~jJo}rcin5oQJ^wpm#eB7 ze~P1~!h1VQUMT z+cw@Dhp0RHO202xzw|-{C%cncLO(7`BdKYJGRZ!#`fL3(1!Ys3)Z+=IXx=+fF6`a=ZCHAU1oGW-k&9*4`l01$Kuusp6qMJI&+Ctst z%N~3GKQU0p^#4^1RCCqks$8%0-C(FES+}DjO@(gl(p31g@LXki&jZIeJ zTmNDH<8U~|viZ6_c`$SK|I-dSrJ=T4*zm7 zaCOUD-Z8+{zdlW;?++%aDS2o=57%&zaxUNMa$zO=5@SnK%|L=c1 zE!ekhyfd@^z~~75B8 zatBS!DV3@RZ~v)HyZKimQGN|Q_V1kmMHDYCJ^Ja_2gL&-4ix-RJAC3$nzHe~{}%$x B)@A?z literal 0 HcmV?d00001 From 72295202ec37a76b90a919e39ae094bb7e56d202 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Sat, 12 Aug 2023 22:19:29 +0200 Subject: [PATCH 41/50] Code cleanup --- .../processor/EtlProcessorApplicationTests.kt | 7 ++++++- .../pseudonym/GpasPseudonymGenerator.java | 4 +--- .../etl/processor/config/AppConfigProperties.kt | 2 +- .../dnpm/etl/processor/pseudonym/extensions.kt | 2 +- .../etl/processor/services/RequestProcessor.kt | 3 --- .../etl/processor/services/ResponseProcessor.kt | 4 +--- .../processor/web/StatisticsRestController.kt | 6 +++--- .../processor/output/RestMtbFileSenderTest.kt | 16 ++++++++-------- .../processor/services/ResponseProcessorTest.kt | 6 +----- .../services/kafka/KafkaResponseProcessorTest.kt | 8 ++++---- 10 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt index 6c5b150..c5a20bb 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt @@ -20,10 +20,13 @@ package dev.dnpm.etl.processor import dev.dnpm.etl.processor.output.MtbFileSender +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.ApplicationContext import org.springframework.test.context.junit.jupiter.SpringExtension import org.testcontainers.junit.jupiter.Testcontainers @@ -34,7 +37,9 @@ import org.testcontainers.junit.jupiter.Testcontainers class EtlProcessorApplicationTests : AbstractTestcontainerTest() { @Test - fun contextLoadsIfMtbFileSenderConfigured() { + fun contextLoadsIfMtbFileSenderConfigured(@Autowired context: ApplicationContext) { + // Simply check bean configuration + assertThat(context).isNotNull } } diff --git a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java index f13a034..732a770 100644 --- a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java +++ b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java @@ -69,13 +69,11 @@ import java.util.HashMap; public class GpasPseudonymGenerator implements Generator { + private final static FhirContext r4Context = FhirContext.forR4(); private final String gPasUrl; private final String psnTargetDomain; - private static FhirContext r4Context = FhirContext.forR4(); private final HttpHeaders httpHeader; - private final RetryTemplate retryTemplate = defaultTemplate(); - private final Logger log = LoggerFactory.getLogger(GpasPseudonymGenerator.class); private SSLContext customSslContext; diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 6502a1b..06e730b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -23,7 +23,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(AppConfigProperties.NAME) data class AppConfigProperties( - var bwhc_uri: String?, + var bwhcUri: String?, var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN ) { companion object { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt index 580785d..c0050a4 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -38,7 +38,7 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) { this.histologyReports.forEach { it.patient = patientPseudonym } this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym } this.molecularPathologyFindings.forEach { it.patient = patientPseudonym } - this.molecularTherapies.forEach { it.history.forEach { it.patient = patientPseudonym } } + this.molecularTherapies.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } } this.ngsReports.forEach { it.patient = patientPseudonym } this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym } this.rebiopsyRequests.forEach { it.patient = patientPseudonym } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 34156f7..3cd912c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -30,7 +30,6 @@ import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith import org.apache.commons.codec.binary.Base32 import org.apache.commons.codec.digest.DigestUtils -import org.slf4j.LoggerFactory import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Service import java.time.Instant @@ -45,8 +44,6 @@ class RequestProcessor( private val applicationEventPublisher: ApplicationEventPublisher ) { - private val logger = LoggerFactory.getLogger(RequestProcessor::class.java) - fun processMtbFile(mtbFile: MtbFile) { val requestId = UUID.randomUUID().toString() val pid = mtbFile.patient.id diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt index f2e9e2e..677443a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -19,7 +19,6 @@ package dev.dnpm.etl.processor.services -import com.fasterxml.jackson.databind.ObjectMapper import dev.dnpm.etl.processor.monitoring.Report import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus @@ -33,8 +32,7 @@ import java.util.* @Service class ResponseProcessor( private val requestRepository: RequestRepository, - private val statisticsUpdateProducer: Sinks.Many, - private val objectMapper: ObjectMapper + private val statisticsUpdateProducer: Sinks.Many ) { private val logger = LoggerFactory.getLogger(ResponseProcessor::class.java) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt index a418772..6f0e820 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt @@ -83,9 +83,9 @@ class StatisticsRestController( .groupBy { formatter.format(it.processedAt) } .map { val requestList = it.value - .groupBy { it.status } - .map { - Pair(it.key, it.value.size) + .groupBy { request -> request.status } + .map { request -> + Pair(request.key, request.value.size) } .toMap() Pair( diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt index 78b5a45..0cad285 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSenderTest.kt @@ -105,7 +105,7 @@ class RestMtbFileSenderTest { } """.trimIndent() - val mtbFile = MtbFile.builder() + val mtbFile: MtbFile = MtbFile.builder() .withPatient( Patient.builder() .withId("PID") @@ -129,7 +129,7 @@ class RestMtbFileSenderTest { ) .build() - private val errorResponseBody = "Sonstiger Fehler bei der Übertragung" + private const val ERROR_RESPONSE_BODY = "Sonstiger Fehler bei der Übertragung" /** * Synthetic http responses with related request status @@ -147,23 +147,23 @@ class RestMtbFileSenderTest { RequestWithResponse( HttpStatus.BAD_REQUEST, "??", - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ), RequestWithResponse( HttpStatus.UNPROCESSABLE_ENTITY, errorBody, - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ), // Some more errors not mentioned in documentation RequestWithResponse( HttpStatus.NOT_FOUND, "what????", - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ), RequestWithResponse( HttpStatus.INTERNAL_SERVER_ERROR, "what????", - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ) ) } @@ -180,12 +180,12 @@ class RestMtbFileSenderTest { RequestWithResponse( HttpStatus.NOT_FOUND, "what????", - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ), RequestWithResponse( HttpStatus.INTERNAL_SERVER_ERROR, "what????", - MtbFileSender.Response(RequestStatus.ERROR, errorResponseBody) + MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY) ) ) } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt index cfb1111..b9e4b7f 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/ResponseProcessorTest.kt @@ -19,8 +19,6 @@ package dev.dnpm.etl.processor.services -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.KotlinModule import dev.dnpm.etl.processor.monitoring.Request import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus @@ -62,12 +60,10 @@ class ResponseProcessorTest { @Mock requestRepository: RequestRepository, @Mock statisticsUpdateProducer: Sinks.Many ) { - val objectMapper = ObjectMapper().registerModule(KotlinModule.Builder().build()) - this.requestRepository = requestRepository this.statisticsUpdateProducer = statisticsUpdateProducer - this.responseProcessor = ResponseProcessor(requestRepository, statisticsUpdateProducer, objectMapper) + this.responseProcessor = ResponseProcessor(requestRepository, statisticsUpdateProducer) } @Test diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt index 0f524ca..6d83146 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt @@ -45,7 +45,7 @@ class KafkaResponseProcessorTest { private lateinit var kafkaResponseProcessor: KafkaResponseProcessor - private fun createkafkaRecord( + private fun createKafkaRecord( requestId: String? = null, statusCode: Int = 200, statusBody: Map? = mapOf() @@ -79,14 +79,14 @@ class KafkaResponseProcessorTest { @Test fun shouldNotProcessRecordsWithoutValidKey() { - this.kafkaResponseProcessor.onMessage(createkafkaRecord(null, 200)) + this.kafkaResponseProcessor.onMessage(createKafkaRecord(null, 200)) verify(eventPublisher, never()).publishEvent(any()) } @Test fun shouldNotProcessRecordsWithoutValidBody() { - this.kafkaResponseProcessor.onMessage(createkafkaRecord(requestId = "TestID1234", statusBody = null)) + this.kafkaResponseProcessor.onMessage(createKafkaRecord(requestId = "TestID1234", statusBody = null)) verify(eventPublisher, never()).publishEvent(any()) } @@ -94,7 +94,7 @@ class KafkaResponseProcessorTest { @ParameterizedTest @MethodSource("statusCodeSource") fun shouldProcessValidRecordsWithStatusCode(statusCode: Int) { - this.kafkaResponseProcessor.onMessage(createkafkaRecord("TestID1234", statusCode)) + this.kafkaResponseProcessor.onMessage(createKafkaRecord("TestID1234", statusCode)) verify(eventPublisher, times(1)).publishEvent(any()) } From 7186a45f6c9e9d6a4919027236450113e4b666b0 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Sat, 12 Aug 2023 22:27:20 +0200 Subject: [PATCH 42/50] Add link to onkostar-plugin-dnpmexport --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a547ab5..48dc27c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisi ### Einordnung innerhalb einer DNPM-ETL-Strecke -Diese Anwendung erlaubt das Entgegennehmen HTTP/REST-Anfragen aus dem Onkostar-Plugin **onkostar-pligin-dnpmexport**. +Diese Anwendung erlaubt das Entgegennehmen HTTP/REST-Anfragen aus dem Onkostar-Plugin **[onkostar-plugin-dnpmexport](https://github.com/CCC-MF/onkostar-plugin-dnpmexport)**. Der Inhalt einer Anfrage, wenn ein bwHC-MTBFile, wird pseudonymisiert und auf Duplikate geprüft. Duplikate werden verworfen, Änderungen werden weitergeleitet. From 2e7ef25a4936ba0ea188cfd9a237ad6be0c6bffe Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Sat, 12 Aug 2023 23:16:17 +0200 Subject: [PATCH 43/50] Update project version and versions in gradle file --- build.gradle.kts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 37fe4e1..8eee6d0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,13 +5,20 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage plugins { war id("org.springframework.boot") version "3.1.2" - id("io.spring.dependency-management") version "1.1.0" + id("io.spring.dependency-management") version "1.1.3" kotlin("jvm") version "1.9.0" kotlin("plugin.spring") version "1.9.0" } group = "de.ukw.ccc" -version = "0.1.1" +version = "0.2.0-SNAPSHOT" + +var versions = mapOf( + "bwhc-dto-java" to "0.2.0", + "hapi-fhir" to "6.6.2", + "httpclient5" to "5.2.1", + "mockito-kotlin" to "5.1.0" +) java { sourceCompatibility = JavaVersion.VERSION_17 @@ -52,10 +59,10 @@ dependencies { implementation("org.flywaydb:flyway-mysql") implementation("commons-codec:commons-codec") implementation("io.projectreactor.kotlin:reactor-kotlin-extensions") - implementation("de.ukw.ccc:bwhc-dto-java:0.2.0") - implementation("ca.uhn.hapi.fhir:hapi-fhir-base:6.6.2") - implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:6.6.2") - implementation("org.apache.httpcomponents.client5:httpclient5:5.2.1") + implementation("de.ukw.ccc:bwhc-dto-java:${versions["bwhc-dto-java"]}") + implementation("ca.uhn.hapi.fhir:hapi-fhir-base:${versions["hapi-fhir"]}") + implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}") + implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") runtimeOnly("org.postgresql:postgresql") developmentOnly("org.springframework.boot:spring-boot-devtools") @@ -64,7 +71,7 @@ dependencies { providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") - testImplementation("org.mockito.kotlin:mockito-kotlin:5.0.0") + testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}") integrationTestImplementation("org.testcontainers:junit-jupiter") integrationTestImplementation("org.testcontainers:postgresql") } From 64b8636145291a3cd28b4354af9ce20e052d672a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 15 Aug 2023 00:49:01 +0200 Subject: [PATCH 44/50] Update Apache Kafka service config for KRaft mode --- dev-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev-compose.yml b/dev-compose.yml index 9b25794..5012bab 100644 --- a/dev-compose.yml +++ b/dev-compose.yml @@ -6,6 +6,12 @@ services: - "9092:9092" environment: ALLOW_PLAINTEXT_LISTENER: "yes" + KAFKA_CFG_NODE_ID: "0" + KAFKA_CFG_PROCESS_ROLES: "controller,broker" + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER mariadb: image: mariadb:10 From 66dc96680da5e263550413493578ebe936dde149 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 15 Aug 2023 01:09:22 +0200 Subject: [PATCH 45/50] Update dev config and added related information into README.md --- README.md | 22 +++++++++++++++++++++- dev-compose.yml | 3 +++ src/main/resources/application-dev.yml | 7 +++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48dc27c..805514f 100644 --- a/README.md +++ b/README.md @@ -79,4 +79,24 @@ für HTTP nicht gibt. ## Docker-Images -Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor \ No newline at end of file +Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor + +## Entwicklungssetup + +Zum Starten einer lokalen Entwicklungs- und Testumgebung kann die beiliegende Datei `dev-compose.yml` verwendet werden. +Diese kann zur Nutzung der Datenbanken **MariaDB** als auch **PostgreSQL** angepasst werden. + +Zur Nutzung von Apache Kafka muss dazu ein Eintrag im hosts-File vorgenommen werden und der Hostname `kafka` auf die lokale +IP-Adresse verweisen. Ohne diese Einstellung ist eine Nutzung von Apache Kafka außerhalb der Docker-Umgebung nicht möglich. + +Beim Start der Anwendung mit dem Profil `dev` wird die in `dev-compose.yml` definierte Umgebung beim Start der +Anwendung mit gestartet: + +``` +SPRING_PROFILES_ACTIVE=dev ./gradlew bootRun +``` + +Die Datei `application-dev.yml` enthält hierzu die Konfiguration für das Profil `dev`. + +Beim Ausführen der Integrationstests wird eine Testdatenbank in einem Docker-Container gestartet. +Siehe hier auch die Klasse `AbstractTestcontainerTest` unter `src/integrationTest`. \ No newline at end of file diff --git a/dev-compose.yml b/dev-compose.yml index 5012bab..8f0780f 100644 --- a/dev-compose.yml +++ b/dev-compose.yml @@ -1,4 +1,7 @@ services: + + # Note: Make sure, hostname "kafka" points to 127.0.0.1 + # otherwise connection will not be available kafka: image: bitnami/kafka hostname: kafka diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 551f3f8..b1cc2fc 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,8 +4,11 @@ spring: file: ./dev-compose.yml app: - rest: - uri: http://localhost:9000/bwhc/etl/api + #rest: + # uri: http://localhost:9000/bwhc/etl/api + + # Note: Make sure, hostname "kafka" points to 127.0.0.1 + # otherwise connection will not be available kafka: topic: test response-topic: test-response From 78b228716396d6e761d08a11a846deb83bdc2e50 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 15 Aug 2023 08:51:23 +0200 Subject: [PATCH 46/50] Add information about Kafka retention time --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 805514f..58092ba 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,19 @@ Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert Lässt sich keine Verbindung zu dem bwHC-Backend aufbauen, wird eine Rückantwort mit Status-Code `900` erwartet, welchen es für HTTP nicht gibt. +#### Retention Time + +Generell werden in Apache Kafka alle Records entsprechend der Konfiguration vorgehalten. +So wird ohne spezielle Konfiguration ein Record für 7 Tage in Apache Kafka gespeichert. +Es sind innerhalb dieses Zeitraums auch alte Informationen weiterhin enthalten, wenn der Consent später abgelehnt wurde. + +Durch eine entsprechende Konfiguration des Topics kann dies verhindert werden. + +Beispiel - auszuführen innerhalb des Kafka-Containers: Löschen alter Records nach einem Tag +``` +kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config retention.ms=86400000 +``` + ## Docker-Images Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor From 2eb5cc61b9809523d51d1fa7af7a1afc1fdb7f0c Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 15 Aug 2023 10:58:17 +0200 Subject: [PATCH 47/50] Change Kafka response body JSON alias --- .../etl/processor/services/kafka/KafkaResponseProcessor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index ef880f4..68e3611 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -84,7 +84,7 @@ class KafkaResponseProcessor( data class ResponseKey(val requestId: String) data class ResponseBody( - @JsonProperty("status_code") @JsonAlias("status code") val statusCode: Int, - @JsonProperty("status_body") val statusBody: Map + @JsonProperty("status_code") @JsonAlias("statusCode") val statusCode: Int, + @JsonProperty("status_body") @JsonAlias("statusBody") val statusBody: Map ) } \ No newline at end of file From 8dc82225a4cd45a315fac3efe4d76513e6d536fc Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 16 Aug 2023 15:25:46 +0200 Subject: [PATCH 48/50] Issue #7: Send and expect requestId in record body, not in record key (#8) --- .../processor/output/KafkaMtbFileSender.kt | 12 ++-- .../processor/services/ResponseProcessor.kt | 2 +- .../services/kafka/KafkaResponseProcessor.kt | 58 ++++++++----------- src/main/resources/application-dev.yml | 2 +- .../output/KafkaMtbFileSenderTest.kt | 12 ++-- .../kafka/KafkaResponseProcessorTest.kt | 54 +++++++++++++---- 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index e7f9769..5772faf 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -40,7 +40,7 @@ class KafkaMtbFileSender( val result = kafkaTemplate.send( kafkaTargetProperties.topic, key(request), - objectMapper.writeValueAsString(request.mtbFile) + objectMapper.writeValueAsString(Data(request.requestId, request.mtbFile)) ) if (result.get() != null) { logger.debug("Sent file via KafkaMtbFileSender") @@ -68,7 +68,7 @@ class KafkaMtbFileSender( val result = kafkaTemplate.send( kafkaTargetProperties.topic, key(request), - objectMapper.writeValueAsString(dummyMtbFile) + objectMapper.writeValueAsString(Data(request.requestId, dummyMtbFile)) ) if (result.get() != null) { @@ -85,12 +85,12 @@ class KafkaMtbFileSender( private fun key(request: MtbFileSender.MtbFileRequest): String { return "{\"pid\": \"${request.mtbFile.patient.id}\", " + - "\"eid\": \"${request.mtbFile.episode.id}\", " + - "\"requestId\": \"${request.requestId}\"}" + "\"eid\": \"${request.mtbFile.episode.id}\"}" } private fun key(request: MtbFileSender.DeleteRequest): String { - return "{\"pid\": \"${request.patientId}\", " + - "\"requestId\": \"${request.requestId}\"}" + return "{\"pid\": \"${request.patientId}\"}" } + + data class Data(val requestId: String, val content: MtbFile) } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt index 677443a..4048348 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -71,7 +71,7 @@ class ResponseProcessor( } else -> { - logger.error("Cannot process response: Unknown response code!") + logger.error("Cannot process response: Unknown response!") return@ifPresentOrElse } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index 68e3611..a29010f 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -41,50 +41,40 @@ class KafkaResponseProcessor( override fun onMessage(data: ConsumerRecord) { try { - Optional.of(objectMapper.readValue(data.key(), ResponseKey::class.java)) + Optional.of(objectMapper.readValue(data.value(), ResponseBody::class.java)) } catch (e: Exception) { + logger.error("Cannot process Kafka response", e) Optional.empty() - }.ifPresentOrElse({ responseKey -> - val event = try { - val responseBody = objectMapper.readValue(data.value(), ResponseBody::class.java) - ResponseEvent( - responseKey.requestId, - Instant.ofEpochMilli(data.timestamp()), - responseBody.statusCode.asRequestStatus(), - when (responseBody.statusCode.asRequestStatus()) { - RequestStatus.SUCCESS -> { - Optional.empty() - } - - RequestStatus.WARNING, RequestStatus.ERROR -> { - Optional.of(objectMapper.writeValueAsString(responseBody.statusBody)) - } - - else -> { - logger.error("Kafka response: Unknown response code!") - Optional.empty() - } + }.ifPresentOrElse({ responseBody -> + val event = ResponseEvent( + responseBody.requestId, + Instant.ofEpochMilli(data.timestamp()), + responseBody.statusCode.asRequestStatus(), + when (responseBody.statusCode.asRequestStatus()) { + RequestStatus.SUCCESS -> { + Optional.empty() } - ) - } catch (e: Exception) { - logger.error("Cannot process Kafka response", e) - ResponseEvent( - responseKey.requestId, - Instant.ofEpochMilli(data.timestamp()), - RequestStatus.ERROR, - Optional.of("Cannot process Kafka response") - ) - } + + RequestStatus.WARNING, RequestStatus.ERROR -> { + Optional.of(objectMapper.writeValueAsString(responseBody.statusBody)) + } + + else -> { + logger.error("Kafka response: Unknown response code '{}'!", responseBody.statusCode) + Optional.empty() + } + } + ) eventPublisher.publishEvent(event) }, { - logger.error("No response key in Kafka response") + logger.error("No requestId in Kafka response") }) } - data class ResponseKey(val requestId: String) - data class ResponseBody( + @JsonProperty("request_id") @JsonAlias("requestId") val requestId: String, @JsonProperty("status_code") @JsonAlias("statusCode") val statusCode: Int, @JsonProperty("status_body") @JsonAlias("statusBody") val statusBody: Map ) + } \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b1cc2fc..a60cd8a 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -11,7 +11,7 @@ app: # otherwise connection will not be available kafka: topic: test - response-topic: test-response + response-topic: test_response servers: kafka:9092 server: diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt index 14bdd5d..3ec9757 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt @@ -97,9 +97,9 @@ class KafkaMtbFileSenderTest { val captor = argumentCaptor() verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture()) assertThat(captor.firstValue).isNotNull - assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"eid\": \"1\", \"requestId\": \"TestID\"}") + assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"eid\": \"1\"}") assertThat(captor.secondValue).isNotNull - assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(mtbFile(Consent.Status.ACTIVE))) + assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData("TestID", Consent.Status.ACTIVE))) } @Test @@ -113,9 +113,9 @@ class KafkaMtbFileSenderTest { val captor = argumentCaptor() verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture()) assertThat(captor.firstValue).isNotNull - assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"requestId\": \"TestID\"}") + assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\"}") assertThat(captor.secondValue).isNotNull - assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(mtbFile(Consent.Status.REJECTED))) + assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData("TestID", Consent.Status.REJECTED))) } companion object { @@ -154,6 +154,10 @@ class KafkaMtbFileSenderTest { }.build() } + fun kafkaRecordData(requestId: String, consentStatus: Consent.Status): KafkaMtbFileSender.Data { + return KafkaMtbFileSender.Data(requestId, mtbFile(consentStatus)) + } + data class TestData(val requestStatus: RequestStatus, val exception: Throwable? = null) @JvmStatic diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt index 6d83146..95bf41b 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessorTest.kt @@ -46,7 +46,7 @@ class KafkaResponseProcessorTest { private lateinit var kafkaResponseProcessor: KafkaResponseProcessor private fun createKafkaRecord( - requestId: String? = null, + requestId: String, statusCode: Int = 200, statusBody: Map? = mapOf() ): ConsumerRecord { @@ -54,15 +54,11 @@ class KafkaResponseProcessorTest { "test-topic", 0, 0, - if (requestId == null) { - null - } else { - this.objectMapper.writeValueAsString(KafkaResponseProcessor.ResponseKey(requestId)) - }, + null, if (statusBody == null) { "" } else { - this.objectMapper.writeValueAsString(KafkaResponseProcessor.ResponseBody(statusCode, statusBody)) + this.objectMapper.writeValueAsString(KafkaResponseProcessor.ResponseBody(requestId, statusCode, statusBody)) } ) } @@ -78,17 +74,51 @@ class KafkaResponseProcessorTest { } @Test - fun shouldNotProcessRecordsWithoutValidKey() { - this.kafkaResponseProcessor.onMessage(createKafkaRecord(null, 200)) + fun shouldNotProcessRecordsWithoutRequestIdInBody() { + val record = ConsumerRecord( + "test-topic", + 0, + 0, + null, + """ + { + "statusCode": 200, + "statusBody": {} + } + """.trimIndent() + ) - verify(eventPublisher, never()).publishEvent(any()) + this.kafkaResponseProcessor.onMessage(record) + + verify(eventPublisher, never()).publishEvent(any()) } @Test - fun shouldNotProcessRecordsWithoutValidBody() { + fun shouldProcessRecordsWithAliasNames() { + val record = ConsumerRecord( + "test-topic", + 0, + 0, + null, + """ + { + "request_id": "test0123456789", + "status_code": 200, + "status_body": {} + } + """.trimIndent() + ) + + this.kafkaResponseProcessor.onMessage(record) + + verify(eventPublisher, times(1)).publishEvent(any()) + } + + @Test + fun shouldNotProcessRecordsWithoutValidStatusBody() { this.kafkaResponseProcessor.onMessage(createKafkaRecord(requestId = "TestID1234", statusBody = null)) - verify(eventPublisher, never()).publishEvent(any()) + verify(eventPublisher, never()).publishEvent(any()) } @ParameterizedTest From 5bd26b894c3cd08ce8aee75c778083e20abefee9 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 18 Aug 2023 22:15:10 +0200 Subject: [PATCH 49/50] Add information about key based retention config --- README.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 58092ba..12acf52 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,32 @@ Beispiel - auszuführen innerhalb des Kafka-Containers: Löschen alter Records n kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config retention.ms=86400000 ``` +#### Key based Retention + +Möchten Sie hingegen immer nur die letzte Meldung für einen Patienten und eine Erkrankung in Apache Kafka vorhalten, +so ist die nachfolgend genannte Konfiguration der Kafka-Topics hilfreich. + + +* `retention.ms`: Möglichst kurze Zeit in der alte Records noch erhalten bleiben, z.B. 10 Sekunden 10000 +* `cleanup.policy`: Löschen alter Records und Beibehalten des letzten Records zu einem Key [delete,compact] + +Beispiele für ein Topic `test`, hier bitte an die verwendeten Topics anpassen. + +``` +kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config retention.ms=10000 +kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config cleanup.policy=[delete,compact] +``` + +Da als Key eines Records die (pseudonymisierte) Patienten-ID und die (anonymisierte) Erkrankungs-ID verwendet wird, +stehen mit obiger Konfiguration der Kafka-Topics nach 10 Sekunden nur noch der jeweils letzte Eintrag für den entsprechenden +Key zur Verfügung. + +Da der Key sowohl für die Records in Richtung bwHC-Backend für die Rückantwort identisch aufgebaut ist, lassen sich so +auch im Falle eines Consent-Widerspruchs die enthaltenen Daten als auch die Offenlegung durch Verifikationsdaten in der +Antwort effektiv verhindern, da diese nach 10 Sekunden gelöscht werden. +Es steht dann nur noch die jeweils letzten Information zur Verfügung, dass für einen Patienten/eine Erkrankung +ein Consent-Widerspruch erfolgte. + ## Docker-Images Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor @@ -112,4 +138,4 @@ SPRING_PROFILES_ACTIVE=dev ./gradlew bootRun Die Datei `application-dev.yml` enthält hierzu die Konfiguration für das Profil `dev`. Beim Ausführen der Integrationstests wird eine Testdatenbank in einem Docker-Container gestartet. -Siehe hier auch die Klasse `AbstractTestcontainerTest` unter `src/integrationTest`. \ No newline at end of file +Siehe hier auch die Klasse `AbstractTestcontainerTest` unter `src/integrationTest`. From 9921e1e684cbc236ac645d5172a2385fa69e5bbc Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Sat, 19 Aug 2023 11:45:21 +0200 Subject: [PATCH 50/50] Throw PseudonymRequestFailed exception with error message This will throw an exception with error message describing what the error is instead of having a more generic NoSuchElementException to be thrown if Optional.get() has no value after calling findFirst() on an empty stream. --- .../pseudonym/GpasPseudonymGenerator.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java index 732a770..91e465b 100644 --- a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java +++ b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java @@ -108,12 +108,19 @@ public class GpasPseudonymGenerator implements Generator { @NotNull public static String unwrapPseudonym(Parameters gPasPseudonymResult) { - Identifier pseudonym = (Identifier) gPasPseudonymResult.getParameter().stream().findFirst() - .get().getPart().stream().filter(a -> a.getName().equals("pseudonym")).findFirst() - .orElseGet(ParametersParameterComponent::new).getValue(); + final var parameters = gPasPseudonymResult.getParameter().stream().findFirst(); + + if (parameters.isEmpty()) { + throw new PseudonymRequestFailed("Empty HL7 parameters, cannot find first one"); + } + + final var identifier = (Identifier) parameters.get().getPart().stream() + .filter(a -> a.getName().equals("pseudonym")) + .findFirst() + .orElseGet(ParametersParameterComponent::new).getValue(); // pseudonym - return pseudonym.getSystem() + "|" + pseudonym.getValue(); + return identifier.getSystem() + "|" + identifier.getValue(); }