pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00
Code Issues Releases Activity

Add request type to be saved with each request

Browse Source
This commit is contained in:
Paul-Christian Volkmer 2023-08-01 08:39:30 +02:00
parent a1e56f1596
commit 4f7f5e4d89
7 changed files with 83 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt
View File

@ -37,6 +37,7 @@ data class Request(
val pid: String,
val fingerprint: String,
val status: RequestStatus,
val type: RequestType,
val processedAt: Instant = Instant.now(),
@Embedded.Nullable var report: Report? = null
)
@ -57,12 +58,22 @@ interface RequestRepository : CrudRepository<Request, Long> {
fun findByUuidEquals(uuid: String): Optional<Request>
@Query("SELECT count(*) AS count, status FROM request GROUP BY status ORDER BY status, count DESC;")
@Query("SELECT count(*) AS count, status FROM request WHERE type = 'MTB_FILE' GROUP BY status ORDER BY status, count DESC;")
fun countStates(): List<CountedState>
@Query("SELECT count(*) AS count, status FROM (" +
"SELECT status, rank() OVER (PARTITION BY patient_id ORDER BY processed_at DESC) AS rank FROM request WHERE status NOT IN ('DUPLICATION')" +
"SELECT status, rank() OVER (PARTITION BY patient_id ORDER BY processed_at DESC) AS rank FROM request " +
"WHERE type = 'MTB_FILE' AND status NOT IN ('DUPLICATION') " +
") rank WHERE rank = 1 GROUP BY status ORDER BY status, count DESC;")
fun findPatientUniqueStates(): List<CountedState>
@Query("SELECT count(*) AS count, status FROM request WHERE type = 'DELETE' GROUP BY status ORDER BY status, count DESC;")
fun countDeleteStates(): List<CountedState>
@Query("SELECT count(*) AS count, status FROM (" +
"SELECT status, rank() OVER (PARTITION BY patient_id ORDER BY processed_at DESC) AS rank FROM request " +
"WHERE type = 'DELETE'" +
") rank WHERE rank = 1 GROUP BY status ORDER BY status, count DESC;")
fun findPatientUniqueDeleteStates(): List<CountedState>
}

4
src/main/kotlin/dev/dnpm/etl/processor/monitoring/RequestStatus.kt
View File

@ -24,7 +24,5 @@ enum class RequestStatus(val value: String) {
WARNING("warning"),
ERROR("error"),
UNKNOWN("unknown"),
DUPLICATION("duplication"),
DELETE_SUCCESS("delete_success"),
DELETE_ERROR("delete_error")
DUPLICATION("duplication")
}

25
src/main/kotlin/dev/dnpm/etl/processor/monitoring/RequestType.kt Normal file
View File

@ -0,0 +1,25 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.monitoring
enum class RequestType(val value: String) {
MTB_FILE("mtb_file"),
DELETE("delete"),
}

5
src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt
View File

@ -26,6 +26,7 @@ import org.springframework.http.HttpHeaders
import org.springframework.http.MediaType
import org.springframework.web.client.RestClientException
import org.springframework.web.client.RestTemplate
import org.springframework.web.util.UriComponentsBuilder
class RestMtbFileSender(private val restTargetProperties: RestTargetProperties) : MtbFileSender {
@ -68,12 +69,12 @@ class RestMtbFileSender(private val restTargetProperties: RestTargetProperties)
headers.contentType = MediaType.APPLICATION_JSON
val entityReq = HttpEntity(null, headers)
restTemplate.delete(
restTargetProperties.uri!!,
"${restTargetProperties.uri}/${request.patientId}",
entityReq,
String::class.java
)
logger.debug("Sent file via RestMtbFileSender")
MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS)
return MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS)
} catch (e: IllegalArgumentException) {
logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!)
} catch (e: RestClientException) {

49
src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
View File

@ -21,10 +21,7 @@ package dev.dnpm.etl.processor.web
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.*
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import org.apache.commons.codec.binary.Base32
@ -62,6 +59,7 @@ class MtbFileController(
pid = pid,
fingerprint = fingerprint(mtbFile),
status = RequestStatus.DUPLICATION,
type = RequestType.MTB_FILE,
report = Report("Duplikat erkannt - keine Daten weitergeleitet")
)
)
@ -106,6 +104,7 @@ class MtbFileController(
pid = pid,
fingerprint = fingerprint(request.mtbFile),
status = requestStatus,
type = RequestType.MTB_FILE,
report = when (requestStatus) {
RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar")
RequestStatus.WARNING -> Report("Warnungen über mangelhafte Daten",
@ -134,13 +133,37 @@ class MtbFileController(
val patientPseudonym = pseudonymizeService.patientPseudonym(patientId)
val responses = senders.map {
it.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym))
val responseStatus = it.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym))
when (responseStatus.status) {
MtbFileSender.ResponseStatus.SUCCESS -> {
logger.info(
"Sent delete for Patient '{}' using '{}'",
patientPseudonym,
it.javaClass.simpleName
)
}
MtbFileSender.ResponseStatus.ERROR -> {
logger.error(
"Error deleting data for Patient '{}' using '{}'",
patientPseudonym,
it.javaClass.simpleName
)
}
else -> {
logger.error(
"Unknown result on deleting data for Patient '{}' using '{}'",
patientPseudonym,
it.javaClass.simpleName
)
}
}
responseStatus
}
val requestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) {
RequestStatus.DELETE_ERROR
val overallRequestStatus = if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.ERROR)) {
RequestStatus.ERROR
} else if (responses.map { it.status }.contains(MtbFileSender.ResponseStatus.SUCCESS)) {
RequestStatus.DELETE_SUCCESS
RequestStatus.SUCCESS
} else {
RequestStatus.UNKNOWN
}
@ -151,9 +174,10 @@ class MtbFileController(
patientId = patientPseudonym,
pid = patientId,
fingerprint = fingerprint(patientPseudonym),
status = requestStatus,
report = when (requestStatus) {
RequestStatus.DELETE_ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar")
status = overallRequestStatus,
type = RequestType.DELETE,
report = when (overallRequestStatus) {
RequestStatus.ERROR -> Report("Fehler bei der Datenübertragung oder Inhalt nicht verarbeitbar")
RequestStatus.UNKNOWN -> Report("Keine Informationen")
else -> null
}
@ -168,7 +192,8 @@ class MtbFileController(
patientId = "???",
pid = patientId,
fingerprint = "",
status = RequestStatus.DELETE_ERROR,
status = RequestStatus.ERROR,
type = RequestType.DELETE,
report = Report("Fehler bei der Pseudonymisierung")
)
)

2
src/main/resources/db/migration/mariadb/V0_1_1__RequestType.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE request ADD COLUMN type varchar(16) not null;
UPDATE request SET type = 'MTB_FILE';

2
src/main/resources/db/migration/postgresql/V0_1_1__RequestType.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE request ADD COLUMN type varchar(16) not null;
UPDATE request SET type = 'MTB_FILE';