1
0
mirror of https://github.com/pcvolkmer/mv64e-etl-processor synced 2025-09-13 09:02:50 +00:00

refactor: remove obsolete bwHC data model V1.0 (#129)

This commit is contained in:
2025-08-12 23:11:50 +02:00
committed by GitHub
parent bf898e5c25
commit 2e88157893
27 changed files with 286 additions and 1596 deletions

View File

@@ -20,11 +20,11 @@
package dev.dnpm.etl.processor
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.output.BwhcV1MtbFileRequest
import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.pcvolkmer.mv64e.mtb.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
@@ -69,7 +69,7 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
properties = [
"app.pseudonymize.generator=buildin",
"app.consent.service=none",
"app.transformations[0].path=diagnoses[*].icd10.version",
"app.transformations[0].path=diagnoses[*].code.version",
"app.transformations[0].from=2013",
"app.transformations[0].to=2014",
]
@@ -94,36 +94,21 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
fun mtbFileIsTransformed() {
doAnswer {
MtbFileSender.Response(RequestStatus.SUCCESS)
}.whenever(mtbFileSender).send(any<BwhcV1MtbFileRequest>())
}.whenever(mtbFileSender).send(any<DnpmV2MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("TEST_12345678")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("TEST_12345678")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("TEST_12345678")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("TEST_12345678")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.withDiagnoses(
.diagnoses(
listOf(
Diagnosis.builder()
.withId("1234")
.withIcd10(Icd10.builder().withCode("F79.9").withVersion("2013").build())
.build()
MtbDiagnosis.builder()
.id("1234")
.patient(Reference.builder().id("TEST_12345678").build())
.code(Coding.builder().code("F79.9").version("2013").build())
.build(),
)
)
.build()
@@ -137,10 +122,10 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
}
}
val captor = argumentCaptor<BwhcV1MtbFileRequest>()
val captor = argumentCaptor<DnpmV2MtbFileRequest>()
verify(mtbFileSender).send(captor.capture())
assertThat(captor.firstValue.content.diagnoses).hasSize(1).allMatch { diagnosis ->
diagnosis.icd10.version == "2014"
diagnosis.code.version == "2014"
}
}
}

View File

@@ -20,7 +20,6 @@
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.anyValueClass
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.consent.ConsentByMtbFile
@@ -29,6 +28,7 @@ import dev.dnpm.etl.processor.consent.IGetConsent
import dev.dnpm.etl.processor.security.TokenRepository
import dev.dnpm.etl.processor.security.UserRoleRepository
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.*
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
@@ -48,6 +48,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.delete
import org.springframework.test.web.servlet.post
import java.time.Instant
import java.util.*
@WebMvcTest(controllers = [MtbFileRestController::class])
@ExtendWith(value = [MockitoExtension::class, SpringExtension::class])
@@ -93,7 +95,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
@Test
@@ -106,7 +108,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
@Test
@@ -119,7 +121,7 @@ class MtbFileRestControllerTest {
status { isUnauthorized() }
}
verify(requestProcessor, never()).processMtbFile(any<MtbFile>())
verify(requestProcessor, never()).processMtbFile(any<Mtb>())
}
@Test
@@ -132,7 +134,7 @@ class MtbFileRestControllerTest {
status { isForbidden() }
}
verify(requestProcessor, never()).processMtbFile(any<MtbFile>())
verify(requestProcessor, never()).processMtbFile(any<Mtb>())
}
@Test
@@ -180,7 +182,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
@Test
@@ -193,33 +195,26 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
}
companion object {
val mtbFile: MtbFile = MtbFile.builder()
.withPatient(
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("PID")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("PID")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("PID")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("PID").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2023-08-08T02:00:00.00Z"))).build())
.build()
)
)
.build()

View File

@@ -24,7 +24,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties
@ConfigurationProperties(AppConfigProperties.NAME)
data class AppConfigProperties(
var bwhcUri: String?,
var transformations: List<TransformationProperties> = listOf(),
var maxRetryAttempts: Int = 3,
var duplicationDetection: Boolean = true,
@@ -128,8 +127,7 @@ data class GIcsConfigProperties(
data class RestTargetProperties(
val uri: String?,
val username: String?,
val password: String?,
val isBwhc: Boolean = false,
val password: String?
) {
companion object {
const val NAME = "app.rest"

View File

@@ -24,7 +24,6 @@ import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.output.RestBwhcMtbFileSender
import dev.dnpm.etl.processor.output.RestDipMtbFileSender
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
@@ -57,11 +56,6 @@ class AppRestConfiguration {
retryTemplate: RetryTemplate,
reportService: ReportService,
): MtbFileSender {
if (restTargetProperties.isBwhc) {
logger.info("Selected 'RestBwhcMtbFileSender'")
return RestBwhcMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
logger.info("Selected 'RestDipMtbFileSender'")
return RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}

View File

@@ -20,13 +20,13 @@
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.consent.TtpConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.ConsentProvision
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.slf4j.LoggerFactory
import org.springframework.http.MediaType
@@ -40,7 +40,7 @@ class KafkaInputListener(
override fun onMessage(record: ConsumerRecord<String, String>) {
when (guessMimeType(record)) {
MediaType.APPLICATION_JSON_VALUE -> handleBwhcMessage(record)
MediaType.APPLICATION_JSON_VALUE -> handleDnpmV2Message(record)
CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE -> handleDnpmV2Message(record)
else -> {
/* ignore other messages */
@@ -57,8 +57,11 @@ class KafkaInputListener(
return record.headers().headers("contentType")?.firstOrNull()?.value().contentToString()
}
private fun handleBwhcMessage(record: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(record.value(), MtbFile::class.java)
private fun handleDnpmV2Message(record: ConsumerRecord<String, String>) {
// Do not handle DNPM-V2 for now
logger.warn("Ignoring MTB File in DNPM V2 format: Not implemented yet")
val mtbFile = objectMapper.readValue(record.value(), Mtb::class.java)
val patientId = PatientId(mtbFile.patient.id)
val firstRequestIdHeader = record.headers().headers("requestId")?.firstOrNull()
val requestId = if (null != firstRequestIdHeader) {
@@ -67,7 +70,8 @@ class KafkaInputListener(
RequestId("")
}
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
// TODO: Use MV Consent for now - needs to be replaced with proper consent evaluation
if (mtbFile.metadata.modelProjectConsent.provisions.filter { it.type == ConsentProvision.PERMIT }.isNotEmpty()) {
logger.debug("Accepted MTB File for processing")
if (requestId.isBlank()) {
requestProcessor.processMtbFile(mtbFile)
@@ -88,9 +92,4 @@ class KafkaInputListener(
}
}
private fun handleDnpmV2Message(record: ConsumerRecord<String, String>) {
// Do not handle DNPM-V2 for now
logger.warn("Ignoring MTB File in DNPM V2 format: Not implemented yet")
}
}

View File

@@ -19,8 +19,6 @@
package dev.dnpm.etl.processor.input
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.consent.IGetConsent
@@ -46,23 +44,7 @@ class MtbFileRestController(
return ResponseEntity.ok("Test")
}
@PostMapping(consumes = [MediaType.APPLICATION_JSON_VALUE])
fun mtbFile(@RequestBody mtbFile: MtbFile): ResponseEntity<Unit> {
val consentStatusBooleanPair = checkConsentStatus(mtbFile)
val ttpConsentStatus = consentStatusBooleanPair.first
val isConsentOK = consentStatusBooleanPair.second
if (isConsentOK) {
logger.debug("Accepted MTB File (bwHC V1) for processing")
requestProcessor.processMtbFile(mtbFile)
} else {
logger.debug("Accepted MTB File (bwHC V1) and process deletion")
val patientId = PatientId(mtbFile.patient.id)
requestProcessor.processDeletion(patientId, ttpConsentStatus)
}
return ResponseEntity.accepted().build()
}
@PostMapping(consumes = [CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE])
@PostMapping(consumes = [MediaType.APPLICATION_JSON_VALUE, CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON_VALUE])
fun mtbFile(@RequestBody mtbFile: Mtb): ResponseEntity<Unit> {
logger.debug("Accepted MTB File (DNPM V2) for processing")
requestProcessor.processMtbFile(mtbFile)
@@ -76,17 +58,4 @@ class MtbFileRestController(
return ResponseEntity.accepted().build()
}
private fun checkConsentStatus(mtbFile: MtbFile): Pair<TtpConsentStatus, Boolean> {
var ttpConsentStatus = iGetConsent.getTtpBroadConsentStatus(mtbFile.patient.id)
val isConsentOK = (ttpConsentStatus == TtpConsentStatus.UNKNOWN_CHECK_FILE && mtbFile.consent.status == Consent.Status.ACTIVE)
|| ttpConsentStatus == TtpConsentStatus.BROAD_CONSENT_GIVEN
if (ttpConsentStatus == TtpConsentStatus.UNKNOWN_CHECK_FILE && mtbFile.consent.status == Consent.Status.REJECTED) {
// in case ttp check is disabled - we propagate rejected status anyway
ttpConsentStatus = TtpConsentStatus.BROAD_CONSENT_MISSING_OR_REJECTED
}
return Pair(ttpConsentStatus, isConsentOK)
}
}

View File

@@ -128,15 +128,11 @@ class RestConnectionCheckService(
fun check() {
result = try {
val available = restTemplate.getForEntity(
if (restTargetProperties.isBwhc) {
UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString()).path("").toUriString()
} else {
UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString())
.pathSegment("mtb")
.pathSegment("kaplan-meier")
.pathSegment("config")
.toUriString()
},
UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString())
.pathSegment("mtb")
.pathSegment("kaplan-meier")
.pathSegment("config")
.toUriString(),
String::class.java
).statusCode == HttpStatus.OK
@@ -267,4 +263,4 @@ class GIcsConnectionCheckService(
override fun connectionAvailable(): ConnectionCheckResult.GIcsConnectionCheckResult {
return this.result
}
}
}

View File

@@ -20,11 +20,11 @@
package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.pcvolkmer.mv64e.mtb.Mtb
import dev.pcvolkmer.mv64e.mtb.MvhMetadata
import org.apache.kafka.clients.producer.ProducerRecord
import org.slf4j.LoggerFactory
import org.springframework.http.MediaType
@@ -50,9 +50,6 @@ class KafkaMtbFileSender(
objectMapper.writeValueAsString(request)
)
when (request) {
is BwhcV1MtbFileRequest -> record.headers()
.add("contentType", MediaType.APPLICATION_JSON_VALUE.toByteArray())
is DnpmV2MtbFileRequest -> record.headers()
.add(
"contentType",
@@ -75,13 +72,8 @@ class KafkaMtbFileSender(
}
override fun send(request: DeleteRequest): MtbFileSender.Response {
val dummyMtbFile = MtbFile.builder()
.withConsent(
Consent.builder()
.withPatient(request.patientId.value)
.withStatus(Consent.Status.REJECTED)
.build()
)
val dummyMtbFile = Mtb.builder()
.metadata(MvhMetadata())
.build()
return try {
@@ -92,7 +84,7 @@ class KafkaMtbFileSender(
key(request),
// Always use old BwhcV1FileRequest with Consent REJECT
objectMapper.writeValueAsString(
BwhcV1MtbFileRequest(
DnpmV2MtbFileRequest(
request.requestId,
dummyMtbFile
)
@@ -119,7 +111,6 @@ class KafkaMtbFileSender(
private fun key(request: MtbRequest): String {
return when (request) {
is BwhcV1MtbFileRequest -> "{\"pid\": \"${request.content.patient.id}\"}"
is DnpmV2MtbFileRequest -> "{\"pid\": \"${request.content.patient.id}\"}"
is DeleteRequest -> "{\"pid\": \"${request.patientId.value}\"}"
else -> throw IllegalArgumentException("Unsupported request type: ${request::class.simpleName}")

View File

@@ -19,7 +19,6 @@
package dev.dnpm.etl.processor.output
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.pcvolkmer.mv64e.mtb.Mtb
@@ -35,15 +34,6 @@ sealed interface MtbFileRequest<out T> : MtbRequest {
fun patientPseudonym(): PatientPseudonym
}
data class BwhcV1MtbFileRequest(
override val requestId: RequestId,
override val content: MtbFile
) : MtbFileRequest<MtbFile> {
override fun patientPseudonym(): PatientPseudonym {
return PatientPseudonym(content.patient.id)
}
}
data class DnpmV2MtbFileRequest(
override val requestId: RequestId,
override val content: Mtb

View File

@@ -1,51 +0,0 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.output
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.config.RestTargetProperties
import dev.dnpm.etl.processor.monitoring.ReportService
import org.springframework.retry.support.RetryTemplate
import org.springframework.web.client.RestTemplate
import org.springframework.web.util.UriComponentsBuilder
class RestBwhcMtbFileSender(
restTemplate: RestTemplate,
private val restTargetProperties: RestTargetProperties,
retryTemplate: RetryTemplate,
reportService: ReportService,
) : RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) {
override fun sendUrl(): String {
return UriComponentsBuilder
.fromUriString(restTargetProperties.uri.toString())
.pathSegment("MTBFile")
.toUriString()
}
override fun deleteUrl(patientId: PatientPseudonym): String {
return UriComponentsBuilder
.fromUriString(restTargetProperties.uri.toString())
.pathSegment("Patient")
.pathSegment(patientId.value)
.toUriString()
}
}

View File

@@ -108,7 +108,6 @@ abstract class RestMtbFileSender(
val password = restTargetProperties.password
val headers = HttpHeaders()
headers.contentType = when (request) {
is BwhcV1MtbFileRequest -> MediaType.APPLICATION_JSON
is DnpmV2MtbFileRequest -> CustomMediaType.APPLICATION_VND_DNPM_V2_MTB_JSON
else -> MediaType.APPLICATION_JSON
}

View File

@@ -19,217 +19,12 @@
package dev.dnpm.etl.processor.pseudonym
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientId
import dev.pcvolkmer.mv64e.mtb.ModelProjectConsent
import dev.pcvolkmer.mv64e.mtb.Mtb
import dev.pcvolkmer.mv64e.mtb.MvhMetadata
import org.apache.commons.codec.digest.DigestUtils
/** Replaces patient ID with generated patient pseudonym
*
* @param pseudonymizeService The pseudonymizeService to be used
* @return The MTB file containing patient pseudonymes
*/
infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
val patientPseudonym = pseudonymizeService.patientPseudonym(PatientId(this.patient.id)).value
this.episode?.patient = patientPseudonym
this.carePlans?.forEach { it.patient = patientPseudonym }
this.patient.id = patientPseudonym
this.claims?.forEach { it.patient = patientPseudonym }
this.consent?.patient = patientPseudonym
this.claimResponses?.forEach { it.patient = patientPseudonym }
this.diagnoses?.forEach { it.patient = patientPseudonym }
this.ecogStatus?.forEach { it.patient = patientPseudonym }
this.familyMemberDiagnoses?.forEach { it.patient = patientPseudonym }
this.geneticCounsellingRequests?.forEach { it.patient = patientPseudonym }
this.histologyReevaluationRequests?.forEach { it.patient = patientPseudonym }
this.histologyReports?.forEach {
it.patient = patientPseudonym
it.tumorMorphology?.patient = patientPseudonym
}
this.lastGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.molecularPathologyFindings?.forEach { it.patient = patientPseudonym }
this.molecularTherapies?.forEach { molecularTherapy ->
molecularTherapy.history.forEach {
it.patient = patientPseudonym
}
}
this.ngsReports?.forEach { it.patient = patientPseudonym }
this.previousGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.rebiopsyRequests?.forEach { it.patient = patientPseudonym }
this.recommendations?.forEach { it.patient = patientPseudonym }
this.responses?.forEach { it.patient = patientPseudonym }
this.studyInclusionRequests?.forEach { it.patient = patientPseudonym }
this.specimens?.forEach { it.patient = patientPseudonym }
}
/**
* Creates new hash of content IDs with given prefix except for patient IDs
*
* @param pseudonymizeService The pseudonymizeService to be used
* @return The MTB file containing rehashed content IDs
*/
infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService) {
val prefix = pseudonymizeService.prefix()
fun anonymize(id: String): String {
val hash = DigestUtils.sha256Hex("$prefix-$id").substring(0, 41).lowercase()
return "$prefix$hash"
}
this.episode?.apply {
id = id?.let {
anonymize(it)
}
}
this.carePlans?.onEach { carePlan ->
carePlan?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
geneticCounsellingRequest = geneticCounsellingRequest?.let { anonymize(it) }
rebiopsyRequests = rebiopsyRequests.map { it?.let { anonymize(it) } }
recommendations = recommendations.map { it?.let { anonymize(it) } }
studyInclusionRequests = studyInclusionRequests.map { it?.let { anonymize(it) } }
}
}
this.claims?.onEach { claim ->
claim?.apply {
id = id?.let { anonymize(it) }
therapy = therapy?.let { anonymize(it) }
}
}
this.claimResponses?.onEach { claimResponse ->
claimResponse?.apply {
id = id?.let { anonymize(it) }
claim = claim?.let { anonymize(it) }
}
}
this.consent?.apply {
id = id?.let { anonymize(it) }
}
this.diagnoses?.onEach { diagnosis ->
diagnosis?.apply {
id = id?.let { anonymize(it) }
histologyResults = histologyResults?.map { it?.let { anonymize(it) } }
}
}
this.ecogStatus?.onEach { ecogStatus ->
ecogStatus?.apply {
id = id?.let { anonymize(it) }
}
}
this.familyMemberDiagnoses?.onEach { familyMemberDiagnosis ->
familyMemberDiagnosis?.apply {
id = id?.let { anonymize(it) }
}
}
this.geneticCounsellingRequests?.onEach { geneticCounsellingRequest ->
geneticCounsellingRequest?.apply {
id = id?.let { anonymize(it) }
}
}
this.histologyReevaluationRequests?.onEach { histologyReevaluationRequest ->
histologyReevaluationRequest?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.histologyReports?.onEach { histologyReport ->
histologyReport?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
tumorMorphology?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
tumorCellContent?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
}
this.lastGuidelineTherapies?.onEach { lastGuidelineTherapy ->
lastGuidelineTherapy?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
}
}
this.molecularPathologyFindings?.onEach { molecularPathologyFinding ->
molecularPathologyFinding?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.molecularTherapies?.onEach { molecularTherapy ->
molecularTherapy?.apply {
history?.onEach { history ->
history?.apply {
id = id?.let { anonymize(it) }
basedOn = basedOn?.let { anonymize(it) }
}
}
}
}
this.ngsReports?.onEach { ngsReport ->
ngsReport?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
tumorCellContent?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
simpleVariants?.onEach { simpleVariant ->
simpleVariant?.apply {
id = id?.let { anonymize(it) }
}
}
}
}
this.previousGuidelineTherapies?.onEach { previousGuidelineTherapy ->
previousGuidelineTherapy?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
medication.forEach { medication ->
medication?.apply {
id = id?.let { anonymize(it) }
}
}
}
}
this.rebiopsyRequests?.onEach { rebiopsyRequest ->
rebiopsyRequest?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.recommendations?.onEach { recommendation ->
recommendation?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
ngsReport = ngsReport?.let { anonymize(it) }
}
}
this.responses?.onEach { response ->
response?.apply {
id = id?.let { anonymize(it) }
therapy = therapy?.let { anonymize(it) }
}
}
this.studyInclusionRequests?.onEach { studyInclusionRequest ->
studyInclusionRequest?.apply {
id = id?.let { anonymize(it) }
reason = reason?.let { anonymize(it) }
}
}
this.specimens?.onEach { specimen ->
specimen?.apply {
id = id?.let { anonymize(it) }
}
}
}
/** Replaces patient ID with generated patient pseudonym
*
* @since 0.11.0
@@ -353,4 +148,4 @@ fun Mtb.ensureMetaDataIsInitialized() {
infix fun Mtb.addGenomDeTan(pseudonymizeService: PseudonymizeService)
{
this.metadata.transferTan = pseudonymizeService.genomDeTan(PatientId(this.patient.id))
}
}

View File

@@ -20,7 +20,6 @@
package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.consent.TtpConsentStatus
@@ -28,7 +27,10 @@ import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.*
import dev.dnpm.etl.processor.output.DeleteRequest
import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest
import dev.dnpm.etl.processor.output.MtbFileRequest
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.pseudonym.addGenomDeTan
import dev.dnpm.etl.processor.pseudonym.anonymizeContentWith
@@ -42,7 +44,6 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.context.ApplicationEventPublisher
import org.springframework.stereotype.Service
import java.lang.RuntimeException
import java.time.Instant
import java.util.*
@@ -59,17 +60,6 @@ class RequestProcessor(
) {
private var logger: Logger = LoggerFactory.getLogger("RequestProcessor")
fun processMtbFile(mtbFile: MtbFile) {
processMtbFile(mtbFile, randomRequestId())
}
fun processMtbFile(mtbFile: MtbFile, requestId: RequestId) {
val pid = PatientId(mtbFile.patient.id)
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = BwhcV1MtbFileRequest(requestId, transformationService.transform(mtbFile))
saveAndSend(request, pid)
}
fun processMtbFile(mtbFile: Mtb) {
processMtbFile(mtbFile, randomRequestId())
@@ -144,7 +134,6 @@ class RequestProcessor(
private fun <T> isDuplication(pseudonymizedMtbFileRequest: MtbFileRequest<T>): Boolean {
val patientPseudonym = when (pseudonymizedMtbFileRequest) {
is BwhcV1MtbFileRequest -> PatientPseudonym(pseudonymizedMtbFileRequest.content.patient.id)
is DnpmV2MtbFileRequest -> PatientPseudonym(pseudonymizedMtbFileRequest.content.patient.id)
}
@@ -214,7 +203,6 @@ class RequestProcessor(
private fun <T> fingerprint(request: MtbFileRequest<T>): Fingerprint {
return when (request) {
is BwhcV1MtbFileRequest -> fingerprint(objectMapper.writeValueAsString(request.content))
is DnpmV2MtbFileRequest -> fingerprint(objectMapper.writeValueAsString(request.content))
}
}

View File

@@ -22,14 +22,9 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.PathNotFoundException
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.pcvolkmer.mv64e.mtb.Mtb
class TransformationService(private val objectMapper: ObjectMapper, private val transformations: List<Transformation>) {
fun transform(mtbFile: MtbFile): MtbFile {
val json = transform(objectMapper.writeValueAsString(mtbFile))
return objectMapper.readValue(json, MtbFile::class.java)
}
fun transform(mtbFile: Mtb): Mtb {
val json = transform(objectMapper.writeValueAsString(mtbFile))

View File

@@ -20,7 +20,7 @@ spring:
app:
rest:
uri: http://localhost:9000/bwhc/etl/api
uri: http://localhost/api
#kafka:
# topic: test
# response-topic: test_response

View File

@@ -20,12 +20,10 @@
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.consent.TtpConsentStatus
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.*
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
@@ -57,9 +55,20 @@ class KafkaInputListenerTest {
@Test
fun shouldProcessMtbFileRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
val mtbFile = Mtb.builder()
.patient(Patient.builder().id("DUMMY_12345678").build())
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.PERMIT).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.build()
kafkaInputListener.onMessage(
@@ -72,14 +81,25 @@ class KafkaInputListenerTest {
)
)
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>())
}
@Test
fun shouldProcessDeleteRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
val mtbFile = Mtb.builder()
.patient(Patient.builder().id("DUMMY_12345678").build())
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.DENY).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.build()
kafkaInputListener.onMessage(
@@ -100,9 +120,20 @@ class KafkaInputListenerTest {
@Test
fun shouldProcessMtbFileRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
val mtbFile = Mtb.builder()
.patient(Patient.builder().id("DUMMY_12345678").build())
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.PERMIT).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
@@ -122,14 +153,25 @@ class KafkaInputListenerTest {
)
)
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>(), anyValueClass())
verify(requestProcessor, times(1)).processMtbFile(any<Mtb>(), anyValueClass())
}
@Test
fun shouldProcessDeleteRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
val mtbFile = Mtb.builder()
.patient(Patient.builder().id("DUMMY_12345678").build())
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.DENY).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
@@ -148,15 +190,29 @@ class KafkaInputListenerTest {
Optional.empty()
)
)
verify(requestProcessor, times(1)).processDeletion(anyValueClass(), anyValueClass(), eq(
TtpConsentStatus.UNKNOWN_CHECK_FILE))
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(), anyValueClass(), eq(
TtpConsentStatus.UNKNOWN_CHECK_FILE
)
)
}
@Test
fun shouldNotProcessDnpmV2Request() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
val mtbFile = Mtb.builder()
.patient(Patient.builder().id("DUMMY_12345678").build())
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.DENY).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.build()
val headers = RecordHeaders(
@@ -180,8 +236,11 @@ class KafkaInputListenerTest {
Optional.empty()
)
)
verify(requestProcessor, times(0)).processDeletion(anyValueClass(), anyValueClass(), eq(
TtpConsentStatus.UNKNOWN_CHECK_FILE))
verify(requestProcessor, times(0)).processDeletion(
anyValueClass(), anyValueClass(), eq(
TtpConsentStatus.UNKNOWN_CHECK_FILE
)
)
}
}

View File

@@ -20,31 +20,21 @@
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Consent.Status
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.consent.ConsentByMtbFile
import dev.dnpm.etl.processor.consent.GicsConsentService
import dev.dnpm.etl.processor.consent.TtpConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.pcvolkmer.mv64e.mtb.Mtb
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
import org.mockito.Mock
import org.mockito.Mockito.times
import org.mockito.Mockito.verify
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.anyValueClass
import org.mockito.kotlin.whenever
import org.springframework.core.io.ClassPathResource
import org.springframework.http.MediaType
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.delete
import org.springframework.test.web.servlet.post
import org.springframework.test.web.servlet.setup.MockMvcBuilders
@@ -53,219 +43,6 @@ class MtbFileRestControllerTest {
private val objectMapper = ObjectMapper()
@Nested
inner class BwhcRequests {
private lateinit var mockMvc: MockMvc
private lateinit var requestProcessor: RequestProcessor
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
val controller = MtbFileRestController(
requestProcessor,
ConsentByMtbFile()
)
this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build()
}
@Test
fun shouldProcessPostRequest() {
mockMvc.post("/mtbfile") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.ACTIVE))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
fun shouldProcessPostRequestWithRejectedConsent() {
mockMvc.post("/mtbfile") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.REJECTED))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(TtpConsentStatus.BROAD_CONSENT_MISSING_OR_REJECTED)
)
}
@Test
fun shouldProcessDeleteRequest() {
mockMvc.delete("/mtbfile/TEST_12345678").andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(TtpConsentStatus.UNKNOWN_CHECK_FILE)
)
}
}
@Nested
inner class BwhcRequestsCheckConsentViaTtp {
private lateinit var mockMvc: MockMvc
private lateinit var requestProcessor: RequestProcessor
private lateinit var gicsConsentService: GicsConsentService
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor,
@Mock gicsConsentService: GicsConsentService
) {
this.requestProcessor = requestProcessor
val controller = MtbFileRestController(requestProcessor, gicsConsentService)
this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build()
this.gicsConsentService = gicsConsentService
}
@ParameterizedTest
@ValueSource(strings = ["ACTIVE", "REJECTED"])
fun shouldProcessPostRequest(status: String) {
whenever(gicsConsentService.getTtpBroadConsentStatus(any())).thenReturn(TtpConsentStatus.BROAD_CONSENT_GIVEN)
mockMvc.post("/mtbfile") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.valueOf(status)))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@ParameterizedTest
@ValueSource(strings = ["ACTIVE", "REJECTED"])
fun shouldProcessPostRequestWithRejectedConsent(status: String) {
whenever(gicsConsentService.getTtpBroadConsentStatus(any())).thenReturn(TtpConsentStatus.BROAD_CONSENT_MISSING_OR_REJECTED)
mockMvc.post("/mtbfile") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.valueOf(status)))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
// consent status from ttp should override file consent value
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(TtpConsentStatus.BROAD_CONSENT_MISSING_OR_REJECTED)
)
}
@Test
fun shouldProcessDeleteRequest() {
mockMvc.delete("/mtbfile/TEST_12345678").andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(TtpConsentStatus.UNKNOWN_CHECK_FILE)
)
verify(gicsConsentService, times(0)).getTtpBroadConsentStatus(any())
}
}
@Nested
inner class BwhcRequestsWithAlias {
private lateinit var mockMvc: MockMvc
private lateinit var requestProcessor: RequestProcessor
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
val controller = MtbFileRestController(
requestProcessor,
ConsentByMtbFile()
)
this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build()
}
@Test
fun shouldProcessPostRequest() {
mockMvc.post("/mtb") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.ACTIVE))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processMtbFile(any<MtbFile>())
}
@Test
fun shouldProcessPostRequestWithRejectedConsent() {
mockMvc.post("/mtb") {
content = objectMapper.writeValueAsString(bwhcMtbFileContent(Status.REJECTED))
contentType = MediaType.APPLICATION_JSON
}.andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(), org.mockito.kotlin.eq(
TtpConsentStatus.BROAD_CONSENT_MISSING_OR_REJECTED
)
)
}
@Test
fun shouldProcessDeleteRequest() {
mockMvc.delete("/mtb/TEST_12345678").andExpect {
status {
isAccepted()
}
}
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(), org.mockito.kotlin.eq(
TtpConsentStatus.UNKNOWN_CHECK_FILE
)
)
}
}
@Nested
inner class RequestsForDnpmDataModel21 {
@@ -304,15 +81,4 @@ class MtbFileRestControllerTest {
}
}
companion object {
fun bwhcMtbFileContent(consentStatus: Status) = MtbFile.builder().withPatient(
Patient.builder().withId("TEST_12345678").withBirthDate("2000-08-08").withGender(Patient.Gender.MALE)
.build()
).withConsent(
Consent.builder().withId("1").withStatus(consentStatus).withPatient("TEST_12345678").build()
).withEpisode(
Episode.builder().withId("1").withPatient("TEST_12345678").withPeriod(PeriodStart("2023-08-08")).build()
).build()
}
}

View File

@@ -20,8 +20,6 @@
package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
@@ -39,7 +37,6 @@ import org.junit.jupiter.params.provider.MethodSource
import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.*
import org.springframework.http.MediaType
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.kafka.support.SendResult
import org.springframework.retry.policy.SimpleRetryPolicy
@@ -74,20 +71,6 @@ class KafkaMtbFileSenderTest {
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldSendMtbFileRequestAndReturnExpectedState(testData: TestData) {
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
val response = kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
assertThat(response.status).isEqualTo(testData.requestStatus)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldSendDeleteRequestAndReturnExpectedState(testData: TestData) {
@@ -102,66 +85,6 @@ class KafkaMtbFileSenderTest {
assertThat(response.status).isEqualTo(testData.requestStatus)
}
@Test
fun shouldSendMtbFileRequestWithCorrectKeyAndHeaderAndBody() {
doAnswer {
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
val captor = argumentCaptor<ProducerRecord<String, String>>()
verify(kafkaTemplate, times(1)).send(captor.capture())
assertThat(captor.firstValue.key()).isNotNull
assertThat(captor.firstValue.key()).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.firstValue.headers().headers("contentType")).isNotNull
assertThat(captor.firstValue.headers().headers("contentType")?.firstOrNull()?.value()).isEqualTo(MediaType.APPLICATION_JSON_VALUE.toByteArray())
assertThat(captor.firstValue.value()).isNotNull
assertThat(captor.firstValue.value()).isEqualTo(objectMapper.writeValueAsString(bwhcV1kafkaRecordData(TEST_REQUEST_ID, Consent.Status.ACTIVE)))
}
@Test
fun shouldSendDeleteRequestWithCorrectKeyAndBody() {
doAnswer {
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
val captor = argumentCaptor<ProducerRecord<String, String>>()
verify(kafkaTemplate, times(1)).send(captor.capture())
assertThat(captor.firstValue.key()).isNotNull
assertThat(captor.firstValue.key()).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.firstValue.value()).isNotNull
assertThat(captor.firstValue.value()).isEqualTo(objectMapper.writeValueAsString(bwhcV1kafkaRecordData(TEST_REQUEST_ID, Consent.Status.REJECTED)))
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldRetryOnMtbFileKafkaSendError(testData: TestData) {
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {
throw testData.exception
}
completedFuture(SendResult<String, String>(null, null))
}.whenever(kafkaTemplate).send(any<ProducerRecord<String, String>>())
kafkaMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1MtbFile(Consent.Status.ACTIVE)))
val expectedCount = when (testData.exception) {
// OK - No Retry
null -> times(1)
// Request failed - Retry max 3 times
else -> times(3)
}
verify(kafkaTemplate, expectedCount).send(any<ProducerRecord<String, String>>())
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.KafkaMtbFileSenderTest#requestWithResponseSource")
fun shouldRetryOnDeleteKafkaSendError(testData: TestData) {
@@ -276,41 +199,6 @@ class KafkaMtbFileSenderTest {
val TEST_REQUEST_ID = RequestId("TestId")
val TEST_PATIENT_PSEUDONYM = PatientPseudonym("PID")
fun bwhcV1MtbFile(consentStatus: Consent.Status): MtbFile {
return if (consentStatus == Consent.Status.ACTIVE) {
MtbFile.builder()
.withPatient(
Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(consentStatus)
.withPatient("PID")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("PID")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
} else {
MtbFile.builder()
.withConsent(
Consent.builder()
.withStatus(consentStatus)
.withPatient("PID")
.build()
)
}.build()
}
fun dnpmV2MtbFile(): Mtb {
return Mtb().apply {
this.patient = dev.pcvolkmer.mv64e.mtb.Patient().apply {
@@ -334,10 +222,6 @@ class KafkaMtbFileSenderTest {
}
}
fun bwhcV1kafkaRecordData(requestId: RequestId, consentStatus: Consent.Status): MtbRequest {
return BwhcV1MtbFileRequest(requestId, bwhcV1MtbFile(consentStatus))
}
fun dnmpV2kafkaRecordData(requestId: RequestId): MtbRequest {
return DnpmV2MtbFileRequest(requestId, dnpmV2MtbFile())
}

View File

@@ -1,313 +0,0 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.KotlinModule
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.config.RestTargetProperties
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.retry.policy.SimpleRetryPolicy
import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.test.web.client.ExpectedCount
import org.springframework.test.web.client.MockRestServiceServer
import org.springframework.test.web.client.match.MockRestRequestMatchers.*
import org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
import org.springframework.web.client.RestTemplate
class RestBwhcMtbFileSenderTest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var restMtbFileSender: RestMtbFileSender
private var reportService = ReportService(ObjectMapper().registerModule(KotlinModule.Builder().build()))
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/mtbfile", null, null)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestBwhcMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
@ParameterizedTest
@MethodSource("deleteRequestWithResponseSource")
fun shouldReturnExpectedResponseForDelete(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.DELETE))
.andExpect(requestTo("http://localhost:9000/mtbfile/Patient/${TEST_PATIENT_PSEUDONYM.value}"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("mtbFileRequestWithResponseSource")
fun shouldReturnExpectedResponseForMtbFilePost(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/mtbfile/MTBFile"))
.andExpect(header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("mtbFileRequestWithResponseSource")
fun shouldRetryOnMtbFileHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/mtbfile", null, null)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestBwhcMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
val expectedCount = when (requestWithResponse.httpStatus) {
// OK - No Retry
HttpStatus.OK, HttpStatus.CREATED -> ExpectedCount.max(1)
// Request failed - Retry max 3 times
else -> ExpectedCount.max(3)
}
this.mockRestServiceServer
.expect(expectedCount, method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/mtbfile/MTBFile"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("deleteRequestWithResponseSource")
fun shouldRetryOnDeleteHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/mtbfile", null, null)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestBwhcMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
val expectedCount = when (requestWithResponse.httpStatus) {
// OK - No Retry
HttpStatus.OK, HttpStatus.CREATED -> ExpectedCount.max(1)
// Request failed - Retry max 3 times
else -> ExpectedCount.max(3)
}
this.mockRestServiceServer
.expect(expectedCount, method(HttpMethod.DELETE))
.andExpect(requestTo("http://localhost:9000/mtbfile/Patient/${TEST_PATIENT_PSEUDONYM.value}"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(DeleteRequest(TEST_REQUEST_ID, TEST_PATIENT_PSEUDONYM))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
companion object {
data class RequestWithResponse(
val httpStatus: HttpStatus,
val body: String,
val response: MtbFileSender.Response
)
val TEST_REQUEST_ID = RequestId("TestId")
val TEST_PATIENT_PSEUDONYM = PatientPseudonym("PID")
val mtbFile: MtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("PID")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("PID")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
private const val ERROR_RESPONSE_BODY = "Sonstiger Fehler bei der Übertragung"
/**
* Synthetic http responses with related request status
* Also see: https://ibmi-intra.cs.uni-tuebingen.de/display/ZPM/bwHC+REST+API
*/
@JvmStatic
fun mtbFileRequestWithResponseSource(): Set<RequestWithResponse> {
return setOf(
RequestWithResponse(
HttpStatus.OK,
responseBodyWithMaxSeverity(ReportService.Severity.INFO),
MtbFileSender.Response(
RequestStatus.SUCCESS,
responseBodyWithMaxSeverity(ReportService.Severity.INFO)
)
),
RequestWithResponse(
HttpStatus.CREATED,
responseBodyWithMaxSeverity(ReportService.Severity.WARNING),
MtbFileSender.Response(
RequestStatus.WARNING,
responseBodyWithMaxSeverity(ReportService.Severity.WARNING)
)
),
RequestWithResponse(
HttpStatus.BAD_REQUEST,
responseBodyWithMaxSeverity(ReportService.Severity.ERROR),
MtbFileSender.Response(RequestStatus.ERROR, responseBodyWithMaxSeverity(ReportService.Severity.ERROR))
),
RequestWithResponse(
HttpStatus.UNPROCESSABLE_ENTITY,
responseBodyWithMaxSeverity(ReportService.Severity.FATAL),
MtbFileSender.Response(
RequestStatus.ERROR,
responseBodyWithMaxSeverity(ReportService.Severity.FATAL)
)
),
// Some more errors not mentioned in documentation
RequestWithResponse(
HttpStatus.NOT_FOUND,
ERROR_RESPONSE_BODY,
MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY)
),
RequestWithResponse(
HttpStatus.INTERNAL_SERVER_ERROR,
ERROR_RESPONSE_BODY,
MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY)
)
)
}
/**
* Synthetic http responses with related request status
* Also see: https://ibmi-intra.cs.uni-tuebingen.de/display/ZPM/bwHC+REST+API
*/
@JvmStatic
fun deleteRequestWithResponseSource(): Set<RequestWithResponse> {
return setOf(
RequestWithResponse(HttpStatus.OK, "", MtbFileSender.Response(RequestStatus.SUCCESS)),
// Some more errors not mentioned in documentation
RequestWithResponse(
HttpStatus.NOT_FOUND,
"what????",
MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY)
),
RequestWithResponse(
HttpStatus.INTERNAL_SERVER_ERROR,
"what????",
MtbFileSender.Response(RequestStatus.ERROR, ERROR_RESPONSE_BODY)
)
)
}
fun responseBodyWithMaxSeverity(severity: ReportService.Severity): String {
return when (severity) {
ReportService.Severity.INFO -> """
{
"patient": "PID",
"issues": [
{ "severity": "info", "message": "Info Message" }
]
}
"""
ReportService.Severity.WARNING -> """
{
"patient": "PID",
"issues": [
{ "severity": "info", "message": "Info Message" },
{ "severity": "warning", "message": "Warning Message" }
]
}
"""
ReportService.Severity.ERROR -> """
{
"patient": "PID",
"issues": [
{ "severity": "info", "message": "Info Message" },
{ "severity": "warning", "message": "Warning Message" },
{ "severity": "error", "message": "Error Message" }
]
}
"""
ReportService.Severity.FATAL -> """
{
"patient": "PID",
"issues": [
{ "severity": "info", "message": "Info Message" },
{ "severity": "warning", "message": "Warning Message" },
{ "severity": "error", "message": "Error Message" },
{ "severity": "fatal", "message": "Fatal Message" }
]
}
"""
}
}
}
}

View File

@@ -21,8 +21,6 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.KotlinModule
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.CustomMediaType
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.RequestId
@@ -54,78 +52,6 @@ import java.util.*
class RestDipMtbFileSenderTest {
@Nested
inner class BwhcV1ContentRequest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var restMtbFileSender: RestMtbFileSender
private var reportService = ReportService(ObjectMapper().registerModule(KotlinModule.Builder().build()))
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#mtbFileRequestWithResponseSource")
fun shouldReturnExpectedResponseForMtbFilePost(requestWithResponse: RequestWithResponse) {
this.mockRestServiceServer
.expect(method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient-record"))
.andExpect(header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
@ParameterizedTest
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#mtbFileRequestWithResponseSource")
fun shouldRetryOnMtbFileHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val retryTemplate = AppConfiguration().retryTemplate(AppConfigProperties("http://localhost:9000"))
retryTemplate.setBackOffPolicy(NoBackOffPolicy())
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.restMtbFileSender =
RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService)
val expectedCount = when (requestWithResponse.httpStatus) {
// OK - No Retry
HttpStatus.OK, HttpStatus.CREATED, HttpStatus.UNPROCESSABLE_ENTITY, HttpStatus.BAD_REQUEST -> ExpectedCount.max(
1
)
// Request failed - Retry max 3 times
else -> ExpectedCount.max(3)
}
this.mockRestServiceServer
.expect(expectedCount, method(HttpMethod.POST))
.andExpect(requestTo("http://localhost:9000/api/mtb/etl/patient-record"))
.andRespond {
withStatus(requestWithResponse.httpStatus).body(requestWithResponse.body).createResponse(it)
}
val response = restMtbFileSender.send(BwhcV1MtbFileRequest(TEST_REQUEST_ID, bwhcV1mtbFile))
assertThat(response.status).isEqualTo(requestWithResponse.response.status)
assertThat(response.body).isEqualTo(requestWithResponse.response.body)
}
}
@Nested
inner class DnpmV2ContentRequest {
@@ -138,7 +64,7 @@ class RestDipMtbFileSenderTest {
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
@@ -176,7 +102,7 @@ class RestDipMtbFileSenderTest {
@BeforeEach
fun setup() {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null)
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
@@ -204,8 +130,8 @@ class RestDipMtbFileSenderTest {
@MethodSource("dev.dnpm.etl.processor.output.RestDipMtbFileSenderTest#deleteRequestWithResponseSource")
fun shouldRetryOnDeleteHttpRequestError(requestWithResponse: RequestWithResponse) {
val restTemplate = RestTemplate()
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null, false)
val retryTemplate = AppConfiguration().retryTemplate(AppConfigProperties("http://localhost:9000"))
val restTargetProperties = RestTargetProperties("http://localhost:9000/api", null, null)
val retryTemplate = AppConfiguration().retryTemplate(AppConfigProperties())
retryTemplate.setBackOffPolicy(NoBackOffPolicy())
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
@@ -245,30 +171,6 @@ class RestDipMtbFileSenderTest {
val TEST_REQUEST_ID = RequestId("TestId")
val TEST_PATIENT_PSEUDONYM = PatientPseudonym("PID")
val bwhcV1mtbFile: MtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("PID")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("PID")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("PID")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
fun dnpmV2MtbFile(): Mtb {
return Mtb().apply {
this.patient = dev.pcvolkmer.mv64e.mtb.Patient().apply {

View File

@@ -21,8 +21,6 @@ package dev.dnpm.etl.processor.pseudonym
import ca.uhn.fhir.context.FhirContext
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.config.GIcsConfigProperties
import dev.dnpm.etl.processor.config.JacksonConfig
@@ -51,172 +49,6 @@ class ExtensionsTest {
return JacksonConfig().objectMapper()
}
@Nested
inner class UsingBwhcDatamodel {
val FAKE_MTB_FILE_PATH = "fake_MTBFile.json"
val CLEAN_PATIENT_ID = "5dad2f0b-49c6-47d8-a952-7b9e9e0f7549"
private fun fakeMtbFile(): MtbFile {
val mtbFile = ClassPathResource(FAKE_MTB_FILE_PATH).inputStream
return getObjectMapper().readValue(mtbFile, MtbFile::class.java)
}
private fun MtbFile.serialized(): String {
return getObjectMapper().writeValueAsString(this)
}
@Test
fun shouldNotContainCleanPatientId(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
val mtbFile = fakeMtbFile()
mtbFile.pseudonymizeWith(pseudonymizeService)
assertThat(mtbFile.patient.id).isEqualTo("PSEUDO-ID")
assertThat(mtbFile.serialized()).doesNotContain(CLEAN_PATIENT_ID)
}
@Test
fun shouldNotContainAnyUuidAfterRehashingOfIds(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = fakeMtbFile()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
val pattern =
"\"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\"".toRegex()
.toPattern()
val matcher = pattern.matcher(mtbFile.serialized())
assertThrows<IllegalStateException> {
matcher.find()
matcher.group()
}.also {
assertThat(it.message).isEqualTo("No match found")
}
}
@Test
fun shouldRehashIdsWithPrefix(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id)
// TESTDOMAIN<sha256(TESTDOMAIN-1)[0-41]>
.isEqualTo("TESTDOMAIN44e20a53bbbf9f3ae39626d05df7014dcd77d6098")
}
@Test
fun shouldNotThrowExceptionOnNullValues(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(anyValueClass())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.withClaims(null)
.withDiagnoses(null)
.withCarePlans(null)
.withClaimResponses(null)
.withEcogStatus(null)
.withFamilyMemberDiagnoses(null)
.withGeneticCounsellingRequests(null)
.withHistologyReevaluationRequests(null)
.withHistologyReports(null)
.withLastGuidelineTherapies(null)
.withMolecularPathologyFindings(null)
.withMolecularTherapies(null)
.withNgsReports(null)
.withPreviousGuidelineTherapies(null)
.withRebiopsyRequests(null)
.withRecommendations(null)
.withResponses(null)
.withStudyInclusionRequests(null)
.withSpecimens(null)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id).isNotNull()
}
}
@Nested
inner class UsingDnpmV2Datamodel {
@@ -251,7 +83,7 @@ class ExtensionsTest {
private fun addConsentData(mtbFile: Mtb) {
val gIcsConfigProperties = GIcsConfigProperties("", "", "")
val appConfigProperties = AppConfigProperties(null, emptyList())
val appConfigProperties = AppConfigProperties(emptyList())
val bundle = Bundle()
val dummyConsent = ConsentProcessorTest.getDummyGenomDeConsent()

View File

@@ -19,8 +19,8 @@
package dev.dnpm.etl.processor.pseudonym
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties
import dev.pcvolkmer.mv64e.mtb.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
@@ -29,31 +29,26 @@ import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.doAnswer
import org.mockito.kotlin.whenever
import java.time.Instant
import java.util.*
@ExtendWith(MockitoExtension::class)
class PseudonymizeServiceTest {
private val mtbFile = MtbFile.builder()
.withPatient(
private val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("123")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("123")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))).build())
.build()
)
)
.build()
@@ -102,4 +97,4 @@ class PseudonymizeServiceTest {
assertThat(tans.add(tan)).`as`("never the same result!").isTrue
}
}
}
}

View File

@@ -51,7 +51,7 @@ class ConsentProcessorTest {
this.objectMapper = jacksonConfig.objectMapper()
this.fhirContext = JacksonConfig.fhirContext()
this.gicsConsentService = gicsConsentService
this.appConfigProperties = AppConfigProperties(null, emptyList())
this.appConfigProperties = AppConfigProperties(emptyList())
this.consentProcessor =
ConsentProcessor(
appConfigProperties,
@@ -168,4 +168,4 @@ class ConsentProcessorTest {
.parseResource<Bundle>(Bundle::class.java, bundle)
}
}
}

View File

@@ -20,22 +20,21 @@
package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.Fingerprint
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.PatientPseudonym
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.consent.GicsConsentService
import dev.dnpm.etl.processor.consent.TtpConsentStatus
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.BwhcV1MtbFileRequest
import dev.dnpm.etl.processor.output.DeleteRequest
import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.output.RestMtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.randomRequestId
import dev.pcvolkmer.mv64e.mtb.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
@@ -49,6 +48,7 @@ import org.mockito.kotlin.argumentCaptor
import org.mockito.kotlin.whenever
import org.springframework.context.ApplicationEventPublisher
import java.time.Instant
import java.util.*
@ExtendWith(MockitoExtension::class)
@@ -77,7 +77,7 @@ class RequestProcessorTest {
this.sender = sender
this.requestService = requestService
this.applicationEventPublisher = applicationEventPublisher
this.appConfigProperties = AppConfigProperties(null)
this.appConfigProperties = AppConfigProperties()
this.consentProcessor = consentProcessor
val objectMapper = ObjectMapper()
@@ -102,7 +102,7 @@ class RequestProcessorTest {
randomRequestId(),
PatientPseudonym("TEST_12345678901"),
PatientId("P1"),
Fingerprint("zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga"),
Fingerprint("6vkiti5bk6ikwifpajpt7cygmd3dvw54d6lwfhzlynb3pqtzferq"),
RequestType.MTB_FILE,
RequestStatus.SUCCESS,
Instant.parse("2023-08-08T02:00:00Z")
@@ -119,29 +119,24 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any<MtbFile>())
}.whenever(transformationService).transform(any<Mtb>())
val mtbFile = MtbFile.builder()
.withPatient(
whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true)
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2023-08-08T02:00:00.00Z"))).build())
.build()
)
)
.build()
@@ -161,7 +156,7 @@ class RequestProcessorTest {
randomRequestId(),
PatientPseudonym("TEST_12345678901"),
PatientId("P1"),
Fingerprint("zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga"),
Fingerprint("4gcjwtjjtcczybsljxepdfpkaeusvd7g3vogfqpmphyffyzfx7dq"),
RequestType.MTB_FILE,
RequestStatus.SUCCESS,
Instant.parse("2023-08-08T02:00:00Z")
@@ -178,29 +173,24 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any<MtbFile>())
}.whenever(transformationService).transform(any<Mtb>())
val mtbFile = MtbFile.builder()
.withPatient(
whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true)
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))).build())
.build()
)
)
.build()
@@ -233,7 +223,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
}.whenever(sender).send(any<DnpmV2MtbFileRequest>())
doAnswer {
it.arguments[0] as String
@@ -241,29 +231,24 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any<MtbFile>())
}.whenever(transformationService).transform(any<Mtb>())
val mtbFile = MtbFile.builder()
.withPatient(
whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true)
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))).build())
.build()
)
)
.build()
@@ -296,7 +281,7 @@ class RequestProcessorTest {
doAnswer {
MtbFileSender.Response(status = RequestStatus.ERROR)
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
}.whenever(sender).send(any<DnpmV2MtbFileRequest>())
doAnswer {
it.arguments[0] as String
@@ -304,29 +289,36 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any<MtbFile>())
}.whenever(transformationService).transform(any<Mtb>())
val mtbFile = MtbFile.builder()
.withPatient(
whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true)
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.metadata(
MvhMetadata
.builder()
.modelProjectConsent(
ModelProjectConsent
.builder()
.provisions(
listOf(Provision.builder().type(ConsentProvision.PERMIT).purpose(ModelProjectConsentPurpose.SEQUENCING).build())
).build()
)
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))).build())
.build()
)
)
.build()
@@ -426,33 +418,28 @@ class RequestProcessorTest {
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any<MtbFile>())
}.whenever(transformationService).transform(any<Mtb>())
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<BwhcV1MtbFileRequest>())
}.whenever(sender).send(any<DnpmV2MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true)
val mtbFile = Mtb.builder()
.patient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.id("123")
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
.episodesOfCare(
listOf(
MtbEpisodeOfCare.builder()
.id("1")
.patient(Reference.builder().id("123").build())
.period(PeriodDate.builder().start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))).build())
.build()
)
)
.build()

View File

@@ -19,20 +19,11 @@
package dev.dnpm.etl.processor.services
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.Diagnosis
import de.ukw.ccc.bwhc.dto.Icd10
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.JacksonConfig
import dev.pcvolkmer.mv64e.mtb.ConsentProvision
import dev.pcvolkmer.mv64e.mtb.ModelProjectConsent
import dev.pcvolkmer.mv64e.mtb.ModelProjectConsentPurpose
import dev.pcvolkmer.mv64e.mtb.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import dev.pcvolkmer.mv64e.mtb.Mtb
import dev.pcvolkmer.mv64e.mtb.MvhMetadata
import dev.pcvolkmer.mv64e.mtb.Provision
import org.hl7.fhir.instance.model.api.IBaseResource
import java.time.Instant
import java.util.Date
@@ -45,82 +36,59 @@ class TransformationServiceTest {
fun setup() {
this.service = TransformationService(
JacksonConfig().objectMapper(), listOf(
Transformation.of("consent.status") from Consent.Status.ACTIVE to Consent.Status.REJECTED,
Transformation.of("diagnoses[*].icd10.version") from "2013" to "2014",
Transformation.of("diagnoses[*].code.version") from "2013" to "2014",
)
)
}
@Test
fun shouldTransformMtbFile() {
val mtbFile = MtbFile.builder().withDiagnoses(
val mtbFile = Mtb.builder().diagnoses(
listOf(
Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also {
it.version = "2013"
}).build()
MtbDiagnosis.builder().id("1234").code(Coding.builder().code("F79.9").version("2013").build()).build()
)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual).isNotNull
assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014")
assertThat(actual.diagnoses[0].code.version).isEqualTo("2014")
}
@Test
fun shouldOnlyTransformGivenValues() {
val mtbFile = MtbFile.builder().withDiagnoses(
val mtbFile = Mtb.builder().diagnoses(
listOf(
Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also {
it.version = "2013"
}).build(),
Diagnosis.builder().withId("5678").withIcd10(Icd10("F79.8").also {
it.version = "2019"
}).build()
MtbDiagnosis.builder().id("1234").code(Coding.builder().code("F79.9").version("2013").build()).build(),
MtbDiagnosis.builder().id("1234").code(Coding.builder().code("F79.8").version("2019").build()).build()
)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual).isNotNull
assertThat(actual.diagnoses[0].icd10.code).isEqualTo("F79.9")
assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014")
assertThat(actual.diagnoses[1].icd10.code).isEqualTo("F79.8")
assertThat(actual.diagnoses[1].icd10.version).isEqualTo("2019")
}
@Test
fun shouldTransformMtbFileWithConsentEnum() {
val mtbFile = MtbFile.builder().withConsent(
Consent("123", "456", Consent.Status.ACTIVE)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual.consent).isNotNull
assertThat(actual.consent.status).isEqualTo(Consent.Status.REJECTED)
assertThat(actual.diagnoses[0].code.code).isEqualTo("F79.9")
assertThat(actual.diagnoses[0].code.version).isEqualTo("2014")
assertThat(actual.diagnoses[1].code.code).isEqualTo("F79.8")
assertThat(actual.diagnoses[1].code.version).isEqualTo("2019")
}
@Test
fun shouldTransformConsentValues() {
val mtbFile = MtbFile.builder().withDiagnoses(
val mtbFile = Mtb.builder().diagnoses(
listOf(
Diagnosis.builder().withId("1234").withIcd10(Icd10("F79.9").also {
it.version = "2013"
}).build(),
Diagnosis.builder().withId("5678").withIcd10(Icd10("F79.8").also {
it.version = "2019"
}).build()
MtbDiagnosis.builder().id("1234").code(Coding.builder().code("F79.9").version("2013").build()).build(),
MtbDiagnosis.builder().id("1234").code(Coding.builder().code("F79.8").version("2019").build()).build()
)
).build()
val actual = this.service.transform(mtbFile)
assertThat(actual).isNotNull
assertThat(actual.diagnoses[0].icd10.code).isEqualTo("F79.9")
assertThat(actual.diagnoses[0].icd10.version).isEqualTo("2014")
assertThat(actual.diagnoses[1].icd10.code).isEqualTo("F79.8")
assertThat(actual.diagnoses[1].icd10.version).isEqualTo("2019")
assertThat(actual.diagnoses[0].code.code).isEqualTo("F79.9")
assertThat(actual.diagnoses[0].code.version).isEqualTo("2014")
assertThat(actual.diagnoses[1].code.code).isEqualTo("F79.8")
assertThat(actual.diagnoses[1].code.version).isEqualTo("2019")
}
@Test
@@ -155,5 +123,4 @@ class TransformationServiceTest {
}
}
}