pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00
Code Issues Releases Activity

Added Link to request report

Browse Source
This commit is contained in:
Paul-Christian Volkmer 2023-07-25 18:37:33 +02:00
parent cd46fa7e09
commit 94846deb98
13 changed files with 188 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/kotlin/dev/dnpm/etl/processor/Exceptions.kt Normal file
View File

@ -0,0 +1,22 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor
class NotFoundException : RuntimeException()

15
src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt
View File

@ -20,6 +20,7 @@
package dev.dnpm.etl.processor.monitoring
import org.springframework.data.annotation.Id
import org.springframework.data.relational.core.mapping.Embedded
import org.springframework.data.relational.core.mapping.Table
import org.springframework.data.repository.CrudRepository
import java.time.Instant
@ -30,16 +31,24 @@ typealias RequestId = UUID
@Table("request")
data class Request(
@Id val id: Long? = null,
val uuid: RequestId = RequestId.randomUUID(),
val uuid: String = RequestId.randomUUID().toString(),
val patientId: String,
val pid: String,
val fingerprint: String,
val status: RequestStatus,
val processedAt: Instant = Instant.now()
val processedAt: Instant = Instant.now(),
@Embedded.Nullable var report: Report? = null
)
data class Report(
val description: String,
val dataQualityReport: String = ""
)
interface RequestRepository : CrudRepository<Request, Long> {
fun findByPatientIdOrderByProcessedAtDesc(patientId: String): List<Request>
fun findAllByPatientIdOrderByProcessedAtDesc(patientId: String): List<Request>
fun findByUuidEquals(uuid: String): Optional<Request>
}

6
src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt
View File

@ -31,14 +31,14 @@ class KafkaMtbFileSender(
private val logger = LoggerFactory.getLogger(KafkaMtbFileSender::class.java)
override fun send(mtbFile: MtbFile): MtbFileSender.ResponseStatus {
override fun send(mtbFile: MtbFile): MtbFileSender.Response {
return try {
kafkaTemplate.sendDefault(objectMapper.writeValueAsString(mtbFile))
logger.debug("Sent file via KafkaMtbFileSender")
MtbFileSender.ResponseStatus.UNKNOWN
MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN)
} catch (e: Exception) {
logger.error("An error occured sending to kafka", e)
MtbFileSender.ResponseStatus.ERROR
MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN)
}
}

4
src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt
View File

@ -22,7 +22,9 @@ package dev.dnpm.etl.processor.output
import de.ukw.ccc.bwhc.dto.MtbFile
interface MtbFileSender {
fun send(mtbFile: MtbFile): ResponseStatus
fun send(mtbFile: MtbFile): Response
data class Response(val status: ResponseStatus, val reason: String = "")
enum class ResponseStatus {
SUCCESS,

10
src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt
View File

@ -34,7 +34,7 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties)
private val restTemplate = RestTemplate()
override fun send(mtbFile: MtbFile): MtbFileSender.ResponseStatus {
override fun send(mtbFile: MtbFile): MtbFileSender.Response {
try {
val headers = HttpHeaders()
headers.contentType = MediaType.APPLICATION_JSON
@ -46,13 +46,13 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties)
)
if (!response.statusCode.is2xxSuccessful) {
logger.warn("Error sending to remote system: {}", response.body)
return MtbFileSender.ResponseStatus.ERROR
return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Status-Code: ${response.statusCode.value()}")
}
logger.debug("Sent file via RestMtbFileSender")
return if (response.body?.contains("warning") == true) {
MtbFileSender.ResponseStatus.WARNING
return MtbFileSender.Response(MtbFileSender.ResponseStatus.WARNING, "${response.body}")
} else {
MtbFileSender.ResponseStatus.SUCCESS
return MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS)
}
} catch (e: IllegalArgumentException) {
logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!)
@ -60,7 +60,7 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties)
logger.info(restTargetProperties.uri!!.toString())
logger.error("Cannot send data to remote system", e)
}
return MtbFileSender.ResponseStatus.ERROR
return MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR, "Sonstiger Fehler bei der Übertragung")
}
}

37
src/main/kotlin/dev/dnpm/etl/processor/web/ApplicationControllerAdvice.kt Normal file
View File

@ -0,0 +1,37 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.NotFoundException
import org.springframework.http.HttpStatus
import org.springframework.web.bind.annotation.ControllerAdvice
import org.springframework.web.bind.annotation.ExceptionHandler
import org.springframework.web.bind.annotation.ResponseStatus
@ControllerAdvice
class ApplicationControllerAdvice {
@ExceptionHandler(NotFoundException::class)
@ResponseStatus(HttpStatus.NOT_FOUND)
fun handleNotFoundException(e: NotFoundException): String {
return "errors/404"
}
}

11
src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt
View File

@ -19,10 +19,13 @@
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.NotFoundException
import dev.dnpm.etl.processor.monitoring.RequestId
import dev.dnpm.etl.processor.monitoring.RequestRepository
import org.springframework.stereotype.Controller
import org.springframework.ui.Model
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PathVariable
import org.springframework.web.bind.annotation.RequestMapping
@Controller
@ -39,4 +42,12 @@ class HomeController(
return "index"
}
@GetMapping(path = ["/report/{id}"])
fun report(@PathVariable id: RequestId, model: Model): String {
val request = requestRepository.findByUuidEquals(id.toString()).orElse(null) ?: throw NotFoundException()
model.addAttribute("request", request)
return "report"
}
}

23
src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.web
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.monitoring.RequestStatus
@ -50,7 +51,7 @@ class MtbFileController(
val pseudonymized = pseudonymizeService.pseudonymize(mtbFile)
val lastRequestForPatient =
requestRepository.findByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id).firstOrNull()
requestRepository.findAllByPatientIdOrderByProcessedAtDesc(pseudonymized.patient.id).firstOrNull()
if (null != lastRequestForPatient && lastRequestForPatient.fingerprint == fingerprint(mtbFile)) {
requestRepository.save(
@ -58,7 +59,8 @@ class MtbFileController(
patientId = pseudonymized.patient.id,
pid = pid,
fingerprint = fingerprint(mtbFile),
status = RequestStatus.DUPLICATION
status = RequestStatus.DUPLICATION,
report = Report("Duplikat erkannt - keine Daten weitergeleitet")
)
)
return ResponseEntity.noContent().build()
@ -66,7 +68,7 @@ class MtbFileController(
val responses = senders.map {
val responseStatus = it.send(pseudonymized)
if (responseStatus == MtbFileSender.ResponseStatus.SUCCESS || responseStatus == MtbFileSender.ResponseStatus.WARNING) {
if (responseStatus.status == MtbFileSender.ResponseStatus.SUCCESS || responseStatus.status == MtbFileSender.ResponseStatus.WARNING) {
logger.info(
"Sent file for Patient '{}' using '{}'",
pseudonymized.patient.id,
@ -82,11 +84,11 @@ class MtbFileController(
responseStatus
}
val requestStatus = if (responses.contains(MtbFileSender.ResponseStatus.ERROR)) {
val requestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) {
RequestStatus.ERROR
} else if (responses.contains(MtbFileSender.ResponseStatus.WARNING)) {
} else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.WARNING)) {
RequestStatus.WARNING
} else if (responses.contains(MtbFileSender.ResponseStatus.SUCCESS)) {
} else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.SUCCESS)) {
RequestStatus.SUCCESS
} else {
RequestStatus.UNKNOWN
@ -97,7 +99,14 @@ class MtbFileController(
patientId = pseudonymized.patient.id,
pid = pid,
fingerprint = fingerprint(mtbFile),
status = requestStatus
status = requestStatus,
report = when (requestStatus) {
RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar")
RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten",
responses.joinToString("\n") { it.reason })
RequestStatus.UNKNOWN -> Report("Keine Informationen")
else -> null
}
)
)

16
src/main/resources/db/migration/mariadb/V0_1_0__Init.sql
View File

@ -1,10 +1,12 @@
CREATE TABLE IF NOT EXISTS request
(
id int auto_increment primary key,
uuid varchar(255) not null unique,
patient_id varchar(255) not null,
pid varchar(255) not null,
fingerprint varchar(255) not null,
status varchar(16) not null,
processed_at datetime default utc_timestamp() not null
id int auto_increment primary key,
uuid varchar(255) not null unique,
patient_id varchar(255) not null,
pid varchar(255) not null,
fingerprint varchar(255) not null,
status varchar(16) not null,
processed_at datetime default utc_timestamp() not null,
description varchar(255) default '',
data_quality_report mediumtext default ''
);

16
src/main/resources/db/migration/postgresql/V0_1_0__Init.sql
View File

@ -1,11 +1,13 @@
CREATE TABLE IF NOT EXISTS request
(
id serial,
uuid varchar(255) not null unique,
patient_id varchar(255) not null,
pid varchar(255) not null,
fingerprint varchar(255) not null,
status varchar(16) not null,
processed_at timestamp with time zone default now() not null,
id serial,
uuid varchar(255) not null unique,
patient_id varchar(255) not null,
pid varchar(255) not null,
fingerprint varchar(255) not null,
status varchar(16) not null,
processed_at timestamp with time zone default now() not null,
description varchar(255) default '',
data_quality_report text default '',
PRIMARY KEY (id)
);

15
src/main/resources/templates/errors/404.html Normal file
View File

@ -0,0 +1,15 @@
<!DOCTYPE html>
<html lang="de" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>ETL-Prozessor</title>
<link rel="stylesheet" th:href="@{/style.css}" />
</head>
<body>
<div th:replace="~{fragments.html :: nav}"></div>
<main>
<h1>Nichts gefunden</h1>
</main>
</body>
</html>

5
src/main/resources/templates/index.html
View File

@ -27,7 +27,10 @@
<td th:if="${request.status.value == 'error'}" class="bg-red"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'unknown'}" class="bg-gray"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'duplication'}" class="bg-gray"><small>[[ ${request.status} ]]</small></td>
<td>[[ ${request.uuid} ]]</td>
<td th:if="not ${request.report}">[[ ${request.uuid} ]]</td>
<td th:if="${request.report}">
<a th:href="@{/report/{id}(id=${request.uuid})}">[[ ${request.uuid} ]]</a>
</td>
<td><time th:datetime="${request.processedAt}">[[ ${request.processedAt} ]]</time></td>
<td>[[ ${request.patientId} ]]</td>
</tr>

42
src/main/resources/templates/report.html Normal file
View File

@ -0,0 +1,42 @@
<!DOCTYPE html>
<html lang="de" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>ETL-Prozessor</title>
<link rel="stylesheet" th:href="@{/style.css}" />
</head>
<body>
<div th:replace="~{fragments.html :: nav}"></div>
<main>
<h1>Anfrage <span class="monospace">[[ ${request.uuid} ]]</span></h1>
<table>
<thead>
<tr>
<th>Status</th>
<th>ID</th>
<th>Datum</th>
<th>Patienten-ID</th>
</tr>
</thead>
<tbody>
<tr>
<td th:if="${request.status.value == 'success'}" class="bg-green"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'warning'}" class="bg-yellow"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'error'}" class="bg-red"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'unknown'}" class="bg-gray"><small>[[ ${request.status} ]]</small></td>
<td th:if="${request.status.value == 'duplication'}" class="bg-gray"><small>[[ ${request.status} ]]</small></td>
<td>[[ ${request.uuid} ]]</td>
<td><time th:datetime="${request.processedAt}">[[ ${request.processedAt} ]]</time></td>
<td>[[ ${request.patientId} ]]</td>
</tr>
</tbody>
</table>
<h2 th:text="${request.report.description}"></h2>
<div class="chart monospace" th:text="${request.report.dataQualityReport}"></div>
</main>
<script th:src="@{/scripts.js}"></script>
</body>
</html>