From 94846deb98ccb892a39795a9e8626f7303efd395 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 25 Jul 2023 18:37:33 +0200 Subject: [PATCH] Added Link to request report --- .../dev/dnpm/etl/processor/Exceptions.kt | 22 ++++++++++ .../dnpm/etl/processor/monitoring/Request.kt | 15 +++++-- .../processor/output/KafkaMtbFileSender.kt | 6 +-- .../etl/processor/output/MtbFileSender.kt | 4 +- .../etl/processor/output/RestMtbFileSender.kt | 10 ++--- .../web/ApplicationControllerAdvice.kt | 37 ++++++++++++++++ .../dnpm/etl/processor/web/HomeController.kt | 11 +++++ .../etl/processor/web/MtbFileController.kt | 23 ++++++---- .../db/migration/mariadb/V0_1_0__Init.sql | 16 +++---- .../db/migration/postgresql/V0_1_0__Init.sql | 16 +++---- src/main/resources/templates/errors/404.html | 15 +++++++ src/main/resources/templates/index.html | 5 ++- src/main/resources/templates/report.html | 42 +++++++++++++++++++ 13 files changed, 188 insertions(+), 34 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/Exceptions.kt create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/web/ApplicationControllerAdvice.kt create mode 100644 src/main/resources/templates/errors/404.html create mode 100644 src/main/resources/templates/report.html diff --git a/src/main/kotlin/dev/dnpm/etl/processor/Exceptions.kt b/src/main/kotlin/dev/dnpm/etl/processor/Exceptions.kt new file mode 100644 index 0000000..32d0954 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/Exceptions.kt @@ -0,0 +1,22 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor + +class NotFoundException : RuntimeException() \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt index e1dd267..7955a9d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt @@ -20,6 +20,7 @@ package dev.dnpm.etl.processor.monitoring import org.springframework.data.annotation.Id +import org.springframework.data.relational.core.mapping.Embedded import org.springframework.data.relational.core.mapping.Table import org.springframework.data.repository.CrudRepository import java.time.Instant @@ -30,16 +31,24 @@ typealias RequestId = UUID @Table("request") data class Request( @Id val id: Long? = null, - val uuid: RequestId = RequestId.randomUUID(), + val uuid: String = RequestId.randomUUID().toString(), val patientId: String, val pid: String, val fingerprint: String, val status: RequestStatus, - val processedAt: Instant = Instant.now() + val processedAt: Instant = Instant.now(), + @Embedded.Nullable var report: Report? = null +) + +data class Report( + val description: String, + val dataQualityReport: String = "" ) interface RequestRepository : CrudRepository { - fun findByPatientIdOrderByProcessedAtDesc(patientId: String): List + fun findAllByPatientIdOrderByProcessedAtDesc(patientId: String): List + + fun findByUuidEquals(uuid: String): Optional } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index acec07a..374c0af 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -31,14 +31,14 @@ class KafkaMtbFileSender( private val logger = LoggerFactory.getLogger(KafkaMtbFileSender::class.java) - override fun send(mtbFile: MtbFile): MtbFileSender.ResponseStatus { + override fun send(mtbFile: MtbFile): MtbFileSender.Response { return try { kafkaTemplate.sendDefault(objectMapper.writeValueAsString(mtbFile)) logger.debug("Sent file via KafkaMtbFileSender") - MtbFileSender.ResponseStatus.UNKNOWN + MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } catch (e: Exception) { logger.error("An error occured sending to kafka", e) - MtbFileSender.ResponseStatus.ERROR + MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt index a085f04..d86fd6b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt @@ -22,7 +22,9 @@ package dev.dnpm.etl.processor.output import de.ukw.ccc.bwhc.dto.MtbFile interface MtbFileSender { - fun send(mtbFile: MtbFile): ResponseStatus + fun send(mtbFile: MtbFile): Response + + data class Response(val status: ResponseStatus, val reason: String = "") enum class ResponseStatus { SUCCESS, diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt index d3b58fb..7a2954a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -34,7 +34,7 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) private val restTemplate = RestTemplate() - override fun send(mtbFile: MtbFile): MtbFileSender.ResponseStatus { + override fun send(mtbFile: MtbFile): MtbFileSender.Response { try { val headers = HttpHeaders() headers.contentType = MediaType.APPLICATION_JSON @@ -46,13 +46,13 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) ) if (!response.statusCode.is2xxSuccessful) { logger.warn("Error sending to remote system: {}", response.body) - return MtbFileSender.ResponseStatus.ERROR + return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Status-Code: ${response.statusCode.value()}") } logger.debug("Sent file via RestMtbFileSender") return if (response.body?.contains("warning") == true) { - MtbFileSender.ResponseStatus.WARNING + return MtbFileSender.Response(MtbFileSender.ResponseStatus.WARNING, "${response.body}") } else { - MtbFileSender.ResponseStatus.SUCCESS + return MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) } } catch (e: IllegalArgumentException) { logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) @@ -60,7 +60,7 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) logger.info(restTargetProperties.uri!!.toString()) logger.error("Cannot send data to remote system", e) } - return MtbFileSender.ResponseStatus.ERROR + return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Sonstiger Fehler bei der Übertragung") } } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/ApplicationControllerAdvice.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/ApplicationControllerAdvice.kt new file mode 100644 index 0000000..bdca57e --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/ApplicationControllerAdvice.kt @@ -0,0 +1,37 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.web + +import dev.dnpm.etl.processor.NotFoundException +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.ControllerAdvice +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.ResponseStatus + +@ControllerAdvice +class ApplicationControllerAdvice { + + @ExceptionHandler(NotFoundException::class) + @ResponseStatus(HttpStatus.NOT_FOUND) + fun handleNotFoundException(e: NotFoundException): String { + return "errors/404" + } + +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt index 10fce68..c139bf7 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt @@ -19,10 +19,13 @@ package dev.dnpm.etl.processor.web +import dev.dnpm.etl.processor.NotFoundException +import dev.dnpm.etl.processor.monitoring.RequestId import dev.dnpm.etl.processor.monitoring.RequestRepository import org.springframework.stereotype.Controller import org.springframework.ui.Model import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @Controller @@ -39,4 +42,12 @@ class HomeController( return "index" } + @GetMapping(path = ["/report/{id}"]) + fun report(@PathVariable id: RequestId, model: Model): String { + val request = requestRepository.findByUuidEquals(id.toString()).orElse(null) ?: throw NotFoundException() + model.addAttribute("request", request) + + return "report" + } + } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt index 04c1594..835f3de 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt @@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.web import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.monitoring.Report import dev.dnpm.etl.processor.monitoring.Request import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus @@ -50,7 +51,7 @@ class MtbFileController( val pseudonymized = pseudonymizeService.pseudonymize(mtbFile) val lastRequestForPatient = - requestRepository.findByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id).firstOrNull() + requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id).firstOrNull() if (null != lastRequestForPatient && lastRequestForPatient.fingerprint == fingerprint(mtbFile)) { requestRepository.save( @@ -58,7 +59,8 @@ class MtbFileController( patientId = pseudonymized.patient.id, pid = pid, fingerprint = fingerprint(mtbFile), - status = RequestStatus.DUPLICATION + status = RequestStatus.DUPLICATION, + report = Report("Duplikat erkannt - keine Daten weitergeleitet") ) ) return ResponseEntity.noContent().build() @@ -66,7 +68,7 @@ class MtbFileController( val responses = senders.map { val responseStatus = it.send(pseudonymized) - if (responseStatus == MtbFileSender.ResponseStatus.SUCCESS || responseStatus == MtbFileSender.ResponseStatus.WARNING) { + if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) { logger.info( "Sent file for Patient '{}' using '{}'", pseudonymized.patient.id, @@ -82,11 +84,11 @@ class MtbFileController( responseStatus } - val requestStatus = if (responses.contains(MtbFileSender.ResponseStatus.ERROR)) { + val requestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) { RequestStatus.ERROR - } else if (responses.contains(MtbFileSender.ResponseStatus.WARNING)) { + } else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.WARNING)) { RequestStatus.WARNING - } else if (responses.contains(MtbFileSender.ResponseStatus.SUCCESS)) { + } else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.SUCCESS)) { RequestStatus.SUCCESS } else { RequestStatus.UNKNOWN @@ -97,7 +99,14 @@ class MtbFileController( patientId = pseudonymized.patient.id, pid = pid, fingerprint = fingerprint(mtbFile), - status = requestStatus + status = requestStatus, + report = when (requestStatus) { + RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar") + RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten", + responses.joinToString("\n") { it.reason }) + RequestStatus.UNKNOWN -> Report("Keine Informationen") + else -> null + } ) ) diff --git a/src/main/resources/db/migration/mariadb/V0_1_0__Init.sql b/src/main/resources/db/migration/mariadb/V0_1_0__Init.sql index 38a3d5f..fa83f8d 100644 --- a/src/main/resources/db/migration/mariadb/V0_1_0__Init.sql +++ b/src/main/resources/db/migration/mariadb/V0_1_0__Init.sql @@ -1,10 +1,12 @@ CREATE TABLE IF NOT EXISTS request ( - id int auto_increment primary key, - uuid varchar(255) not null unique, - patient_id varchar(255) not null, - pid varchar(255) not null, - fingerprint varchar(255) not null, - status varchar(16) not null, - processed_at datetime default utc_timestamp() not null + id int auto_increment primary key, + uuid varchar(255) not null unique, + patient_id varchar(255) not null, + pid varchar(255) not null, + fingerprint varchar(255) not null, + status varchar(16) not null, + processed_at datetime default utc_timestamp() not null, + description varchar(255) default '', + data_quality_report mediumtext default '' ); \ No newline at end of file diff --git a/src/main/resources/db/migration/postgresql/V0_1_0__Init.sql b/src/main/resources/db/migration/postgresql/V0_1_0__Init.sql index 483479d..930064c 100644 --- a/src/main/resources/db/migration/postgresql/V0_1_0__Init.sql +++ b/src/main/resources/db/migration/postgresql/V0_1_0__Init.sql @@ -1,11 +1,13 @@ CREATE TABLE IF NOT EXISTS request ( - id serial, - uuid varchar(255) not null unique, - patient_id varchar(255) not null, - pid varchar(255) not null, - fingerprint varchar(255) not null, - status varchar(16) not null, - processed_at timestamp with time zone default now() not null, + id serial, + uuid varchar(255) not null unique, + patient_id varchar(255) not null, + pid varchar(255) not null, + fingerprint varchar(255) not null, + status varchar(16) not null, + processed_at timestamp with time zone default now() not null, + description varchar(255) default '', + data_quality_report text default '', PRIMARY KEY (id) ); \ No newline at end of file diff --git a/src/main/resources/templates/errors/404.html b/src/main/resources/templates/errors/404.html new file mode 100644 index 0000000..8900433 --- /dev/null +++ b/src/main/resources/templates/errors/404.html @@ -0,0 +1,15 @@ + + + + + ETL-Prozessor + + + +
+
+

Nichts gefunden

+
+ + + \ No newline at end of file diff --git a/src/main/resources/templates/index.html b/src/main/resources/templates/index.html index 8aa60d1..b8a4fb0 100644 --- a/src/main/resources/templates/index.html +++ b/src/main/resources/templates/index.html @@ -27,7 +27,10 @@ [[ ${request.status} ]] [[ ${request.status} ]] [[ ${request.status} ]] - [[ ${request.uuid} ]] + [[ ${request.uuid} ]] + + [[ ${request.uuid} ]] + [[ ${request.patientId} ]] diff --git a/src/main/resources/templates/report.html b/src/main/resources/templates/report.html new file mode 100644 index 0000000..5a08e44 --- /dev/null +++ b/src/main/resources/templates/report.html @@ -0,0 +1,42 @@ + + + + + ETL-Prozessor + + + +
+
+ +

Anfrage [[ ${request.uuid} ]]

+ + + + + + + + + + + + + + + + + + + + + + +
StatusIDDatumPatienten-ID
[[ ${request.status} ]][[ ${request.status} ]][[ ${request.status} ]][[ ${request.status} ]][[ ${request.status} ]][[ ${request.uuid} ]][[ ${request.patientId} ]]
+ +

+
+
+ + + \ No newline at end of file