build: bump version

feat #15: add connection checks to bwHC backend

build.gradle.kts

@@ -4,26 +4,22 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 
 plugins {
     war
-    id("org.springframework.boot") version "3.1.6"
+    id("org.springframework.boot") version "3.2.1"
     id("io.spring.dependency-management") version "1.1.4"
-    kotlin("jvm") version "1.9.21"
+    kotlin("jvm") version "1.9.22"
-    kotlin("plugin.spring") version "1.9.21"
+    kotlin("plugin.spring") version "1.9.22"
 }
 
 group = "de.ukw.ccc"
-version = "0.2.0-SNAPSHOT"
+version = "0.4.0"
 
 var versions = mapOf(
     "bwhc-dto-java" to "0.2.0",
-    "hapi-fhir" to "6.6.2",
+    "hapi-fhir" to "6.10.2",
     "httpclient5" to "5.2.1",
-    "mockito-kotlin" to "5.1.0"
+    "mockito-kotlin" to "5.2.1"
 )
 
-// Override Apache Kafka to be used
-// Fixes: CVE-2023-34455, CVE-2023-34454, CVE-2023-34453 and CVE-2023-43642
-extra["kafka.version"] = "3.6.0"
-
 java {
     sourceCompatibility = JavaVersion.VERSION_17
 }
@@ -60,8 +56,6 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-data-jdbc")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
     implementation("org.springframework.kafka:spring-kafka")
-    // fix CVE-2022-1471
-    implementation("org.yaml:snakeyaml:2.1")
     implementation("org.flywaydb:flyway-mysql")
     implementation("commons-codec:commons-codec")
     implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")

src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt

@@ -46,6 +46,11 @@ import org.testcontainers.junit.jupiter.Testcontainers
 @ExtendWith(SpringExtension::class)
 @SpringBootTest
 @MockBean(MtbFileSender::class)
+@TestPropertySource(
+    properties = [
+        "app.rest.uri=http://example.com"
+    ]
+)
 class EtlProcessorApplicationTests : AbstractTestcontainerTest() {
 
     @Test

src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt

@@ -32,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.boot.test.mock.mockito.MockBean
+import org.springframework.test.context.TestPropertySource
 import org.springframework.test.context.junit.jupiter.SpringExtension
 import org.springframework.transaction.annotation.Transactional
 import org.testcontainers.junit.jupiter.Testcontainers
@@ -43,6 +44,11 @@ import java.util.*
 @SpringBootTest
 @Transactional
 @MockBean(MtbFileSender::class)
+@TestPropertySource(
+    properties = [
+        "app.rest.uri=http://example.com"
+    ]
+)
 class RequestServiceIntegrationTest : AbstractTestcontainerTest() {
 
     private lateinit var requestRepository: RequestRepository

src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt

@@ -1,7 +1,7 @@
 /*
  * This file is part of ETL-Processor
  *
- * Copyright (c) 2023  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ * Copyright (c) 2024  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published
@@ -32,6 +32,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import org.springframework.scheduling.annotation.EnableScheduling
 import reactor.core.publisher.Sinks
 
 @Configuration
@@ -42,6 +43,7 @@ import reactor.core.publisher.Sinks
         GPasConfigProperties::class
     ]
 )
+@EnableScheduling
 class AppConfiguration {
 
     private val logger = LoggerFactory.getLogger(AppConfiguration::class.java)

src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt

@@ -1,7 +1,7 @@
 /*
  * This file is part of ETL-Processor
  *
- * Copyright (c) 2023  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ * Copyright (c) 2024  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published
@@ -20,6 +20,8 @@
 package dev.dnpm.etl.processor.config
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
+import dev.dnpm.etl.processor.monitoring.KafkaConnectionCheckService
 import dev.dnpm.etl.processor.output.KafkaMtbFileSender
 import dev.dnpm.etl.processor.output.MtbFileSender
 import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor
@@ -76,4 +78,9 @@ class AppKafkaConfiguration {
         return KafkaResponseProcessor(applicationEventPublisher, objectMapper)
     }
 
+    @Bean
+    fun connectionCheckService(consumerFactory: ConsumerFactory<String, String>): ConnectionCheckService {
+        return KafkaConnectionCheckService(consumerFactory.createConsumer())
+    }
+
 }

src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt

@@ -1,7 +1,7 @@
 /*
  * This file is part of ETL-Processor
  *
- * Copyright (c) 2023  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ * Copyright (c) 2024  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published
@@ -19,6 +19,8 @@
 
 package dev.dnpm.etl.processor.config
 
+import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
+import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
 import dev.dnpm.etl.processor.output.MtbFileSender
 import dev.dnpm.etl.processor.output.RestMtbFileSender
 import org.slf4j.LoggerFactory
@@ -54,5 +56,13 @@ class AppRestConfiguration {
         return RestMtbFileSender(restTemplate, restTargetProperties)
     }
 
+    @Bean
+    fun connectionCheckService(
+        restTemplate: RestTemplate,
+        restTargetProperties: RestTargetProperties
+    ): ConnectionCheckService {
+        return RestConnectionCheckService(restTemplate, restTargetProperties)
+    }
+
 }
 

src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt

@@ -0,0 +1,85 @@
+/*
+ * This file is part of ETL-Processor
+ *
+ * Copyright (c) 2024  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+
+package dev.dnpm.etl.processor.monitoring
+
+import dev.dnpm.etl.processor.config.RestTargetProperties
+import jakarta.annotation.PostConstruct
+import org.apache.kafka.clients.consumer.Consumer
+import org.apache.kafka.common.errors.TimeoutException
+import org.springframework.http.HttpStatus
+import org.springframework.scheduling.annotation.Scheduled
+import org.springframework.web.client.RestTemplate
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.toJavaDuration
+
+interface ConnectionCheckService {
+
+    fun connectionAvailable(): Boolean
+
+}
+
+class KafkaConnectionCheckService(
+    private val consumer: Consumer<String, String>
+) : ConnectionCheckService {
+
+    private var connectionAvailable: Boolean = false
+
+
+    @PostConstruct
+    @Scheduled(cron = "0 * * * * *")
+    fun check() {
+        connectionAvailable = try {
+            null != consumer.listTopics(5.seconds.toJavaDuration())
+        } catch (e: TimeoutException) {
+            false
+        }
+    }
+
+    override fun connectionAvailable(): Boolean {
+        return this.connectionAvailable
+    }
+
+}
+
+class RestConnectionCheckService(
+    private val restTemplate: RestTemplate,
+    private val restTargetProperties: RestTargetProperties
+) : ConnectionCheckService {
+
+    private var connectionAvailable: Boolean = false
+
+    @PostConstruct
+    @Scheduled(cron = "0 * * * * *")
+    fun check() {
+        connectionAvailable = try {
+            restTemplate.getForEntity(
+                restTargetProperties.uri?.replace("/etl/api", "").toString(),
+                String::class.java
+            ).statusCode == HttpStatus.OK
+        } catch (e: Exception) {
+            false
+        }
+    }
+
+    override fun connectionAvailable(): Boolean {
+        return this.connectionAvailable
+    }
+}

src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt

@@ -34,7 +34,10 @@ class ReportService(
             return listOf()
         }
         return try {
-            objectMapper.readValue(dataQualityReport, DataQualityReport::class.java).issues
+            objectMapper
+                .readValue(dataQualityReport, DataQualityReport::class.java)
+                .issues
+                .sortedBy { it.severity }
         } catch (e: Exception) {
             val otherIssue =
                 Issue(Severity.ERROR, "Not parsable data quality report '$dataQualityReport'")
@@ -56,5 +59,6 @@ class ReportService(
     enum class Severity(@JsonValue val value: String) {
         ERROR("error"),
         WARNING("warning"),
+        INFO("info")
     }
 }

src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt

@@ -35,7 +35,10 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
     this.familyMemberDiagnoses.forEach { it.patient = patientPseudonym }
     this.geneticCounsellingRequests.forEach { it.patient = patientPseudonym }
     this.histologyReevaluationRequests.forEach { it.patient = patientPseudonym }
-    this.histologyReports.forEach { it.patient = patientPseudonym }
+    this.histologyReports.forEach {
+        it.patient = patientPseudonym
+        it.tumorMorphology.patient = patientPseudonym
+    }
     this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym }
     this.molecularPathologyFindings.forEach { it.patient = patientPseudonym }
     this.molecularTherapies.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
@@ -45,6 +48,6 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
     this.recommendations.forEach { it.patient = patientPseudonym }
     this.recommendations.forEach { it.patient = patientPseudonym }
     this.responses.forEach { it.patient = patientPseudonym }
-    this.specimens.forEach { it.patient = patientPseudonym }
+    this.studyInclusionRequests.forEach { it.patient = patientPseudonym }
     this.specimens.forEach { it.patient = patientPseudonym }
 }

src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt

@@ -19,6 +19,9 @@
 
 package dev.dnpm.etl.processor.web
 
+import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
+import dev.dnpm.etl.processor.output.MtbFileSender
+import dev.dnpm.etl.processor.pseudonym.Generator
 import dev.dnpm.etl.processor.services.TransformationService
 import org.springframework.stereotype.Controller
 import org.springframework.ui.Model
@@ -26,16 +29,23 @@ import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
 
 @Controller
-@RequestMapping(path = ["transformations"])
+@RequestMapping(path = ["configs"])
-class TransformationController(
+class ConfigController(
-    private val transformationService: TransformationService
+    private val transformationService: TransformationService,
+    private val pseudonymGenerator: Generator,
+    private val mtbFileSender: MtbFileSender,
+    private val connectionCheckService: ConnectionCheckService
+
 ) {
 
     @GetMapping
     fun index(model: Model): String {
+        model.addAttribute("pseudonymGenerator", pseudonymGenerator.javaClass.simpleName)
+        model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
+        model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable())
         model.addAttribute("transformations", transformationService.getTransformations())
 
-        return "transformations"
+        return "configs"
     }
 
 }

src/main/resources/static/style.css

@@ -1,6 +1,9 @@
 :root {
     --table-border: rgba(96, 96, 96, 1);
 
+    --bg-blue: rgb(0, 74, 157);
+    --bg-blue-op: rgba(0, 74, 157, .35);
+
     --bg-green: rgb(0, 128, 0);
     --bg-green-op: rgba(0, 128, 0, .35);
 
@@ -181,6 +184,15 @@ td {
     font-family: monospace;
 }
 
+td.bg-blue, th.bg-blue {
+    background: var(--bg-blue);
+    color: white;
+}
+
+tr:has(td.bg-blue) {
+    background: var(--bg-blue-op);
+}
+
 td.bg-green, th.bg-green {
     background: var(--bg-green);
     color: white;

src/main/resources/templates/configs.html

@@ -8,16 +8,45 @@
 <body>
     <div th:replace="~{fragments.html :: nav}"></div>
     <main>
-        <h1>Transformationen</h1>
+        <h1>Konfiguration</h1>
 
-        <h2>Syntax</h2>
+        <h2>Allgemeine Konfiguration</h2>
+        <table>
+            <thead>
+            <tr>
+                <th>Name</th>
+                <th>Wert</th>
+            </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>Pseudonym erzeugt über</td>
+                    <td>[[ ${pseudonymGenerator} ]]</td>
+                </tr>
+                <tr>
+                    <td>MTBFile-Sender</td>
+                    <td>[[ ${mtbFileSender} ]]</td>
+                </tr>
+            </tbody>
+        </table>
+
+        <h2><span th:if="${connectionAvailable}">✅</span><span th:if="${not(connectionAvailable)}">⚡</span> Verbindung zum bwHC-Backend</h2>
+        <p>
+            Verbindung über <code>[[ ${mtbFileSender} ]]</code>. Die Verbindung ist aktuell
+            <strong th:if="${connectionAvailable}" style="color: green">verfügbar.</strong>
+            <strong th:if="${not(connectionAvailable)}" style="color: red">nicht verfügbar!</strong>
+        </p>
+
+        <h2>Transformationen</h2>
+
+        <h3>Syntax</h3>
         Hier einige Beispiele zum Syntax des JSON-Path
         <ul>
             <li style="padding: 0.6rem 0;"><span class="bg-path">diagnoses[*].icdO3T.version</span>: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung</li>
             <li style="padding: 0.6rem 0;"><span class="bg-path">patient.gender</span>: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format</li>
         </ul>
 
-        <h2>Konfigurierte Transformationen</h2>
+        <h3>Konfigurierte Transformationen</h3>
         <p>
             Hier sehen Sie eine Übersicht der konfigurierten Transformationen.
         </p>

src/main/resources/templates/fragments.html

@@ -10,7 +10,7 @@
             <ul>
                 <li><a th:href="@{/}">Übersicht</a></li>
                 <li><a th:href="@{/statistics}">Statistiken</a></li>
-                <li><a th:href="@{/transformations}">Transformationen</a></li>
+                <li><a th:href="@{/configs}">Konfiguration</a></li>
             </ul>
         </nav>
     </div>

src/main/resources/templates/report.html

@@ -45,6 +45,7 @@
             </thead>
             <tbody>
             <tr th:each="issue : ${issues}">
+                <td th:if="${issue.severity.value == 'info'}" class="bg-blue"><small>[[ ${issue.severity} ]]</small></td>
                 <td th:if="${issue.severity.value == 'warning'}" class="bg-yellow"><small>[[ ${issue.severity} ]]</small></td>
                 <td th:if="${issue.severity.value == 'error'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td>
                 <td>[[ ${issue.message} ]]</td>

src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt

@@ -0,0 +1,64 @@
+/*
+ * This file is part of ETL-Processor
+ *
+ * Copyright (c) 2023  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package dev.dnpm.etl.processor.pseudonym
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import de.ukw.ccc.bwhc.dto.MtbFile
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.mockito.ArgumentMatchers
+import org.mockito.Mock
+import org.mockito.junit.jupiter.MockitoExtension
+import org.mockito.kotlin.doAnswer
+import org.mockito.kotlin.whenever
+import org.springframework.core.io.ClassPathResource
+
+const val FAKE_MTB_FILE_PATH = "fake_MTBFile.json"
+const val CLEAN_PATIENT_ID = "5dad2f0b-49c6-47d8-a952-7b9e9e0f7549"
+
+@ExtendWith(MockitoExtension::class)
+class ExtensionsTest {
+
+    private fun fakeMtbFile(): MtbFile {
+        val mtbFile = ClassPathResource(FAKE_MTB_FILE_PATH).inputStream
+        return ObjectMapper().readValue(mtbFile, MtbFile::class.java)
+    }
+
+    private fun MtbFile.serialized(): String {
+        return ObjectMapper().writeValueAsString(this)
+    }
+
+    @Test
+    fun shouldNotContainCleanPatientId(@Mock pseudonymizeService: PseudonymizeService) {
+        doAnswer {
+            it.arguments[0]
+            "PSEUDO-ID"
+        }.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
+
+        val mtbFile = fakeMtbFile()
+
+        mtbFile.pseudonymizeWith(pseudonymizeService)
+
+        assertThat(mtbFile.patient.id).isEqualTo("PSEUDO-ID")
+        assertThat(mtbFile.serialized()).doesNotContain(CLEAN_PATIENT_ID)
+    }
+
+}

src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt

@@ -41,6 +41,7 @@ class ReportServiceTest {
             {
                 "patient": "4711",
                 "issues": [
+                    { "severity": "info", "message": "Info Message" },
                     { "severity": "warning", "message": "Warning Message" },
                     { "severity": "error", "message": "Error Message" }
                 ]
@@ -49,11 +50,13 @@ class ReportServiceTest {
 
         val actual = this.reportService.deserialize(json)
 
-        assertThat(actual).hasSize(2)
+        assertThat(actual).hasSize(3)
-        assertThat(actual[0].severity).isEqualTo(ReportService.Severity.WARNING)
+        assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR)
-        assertThat(actual[0].message).isEqualTo("Warning Message")
+        assertThat(actual[0].message).isEqualTo("Error Message")
-        assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR)
+        assertThat(actual[1].severity).isEqualTo(ReportService.Severity.WARNING)
-        assertThat(actual[1].message).isEqualTo("Error Message")
+        assertThat(actual[1].message).isEqualTo("Warning Message")
+        assertThat(actual[2].severity).isEqualTo(ReportService.Severity.INFO)
+        assertThat(actual[2].message).isEqualTo("Info Message")
     }
 
     @Test