chore: bump version for new release

This should not occur but if, it should not result in NPE except for

* Patient
* Consent
* Episode

.github/workflows/deploy.yml

@@ -21,7 +21,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -30,6 +30,6 @@ jobs:
       - name: Execute image build and push
         run: |
           ./gradlew bootBuildImage
-          docker tag ghcr.io/ccc-mf/etl-processor ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }}
-          docker push ghcr.io/ccc-mf/etl-processor
-          docker push ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }}
+          docker tag ghcr.io/${{ github.repository }} ghcr.io/${{ github.repository }}:${{ github.ref_name }}
+          docker push ghcr.io/${{ github.repository }}
+          docker push ghcr.io/${{ github.repository }}:${{ github.ref_name }}

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: '21'
           distribution: 'temurin'
@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: '21'
           distribution: 'temurin'

build.gradle.kts

@@ -4,23 +4,23 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 
 plugins {
     war
-    id("org.springframework.boot") version "3.2.3"
-    id("io.spring.dependency-management") version "1.1.4"
-    kotlin("jvm") version "1.9.22"
-    kotlin("plugin.spring") version "1.9.22"
+    id("org.springframework.boot") version "3.2.11"
+    id("io.spring.dependency-management") version "1.1.5"
+    kotlin("jvm") version "1.9.24"
+    kotlin("plugin.spring") version "1.9.24"
 }
 
-group = "de.ukw.ccc"
-version = "0.9.0"
+group = "dev.dnpm"
+version = "0.9-SNAPSHOT"
 
 var versions = mapOf(
-    "bwhc-dto-java" to "0.2.0",
-    "hapi-fhir" to "6.10.2",
-    "httpclient5" to "5.2.1",
-    "mockito-kotlin" to "5.2.1",
+    "bwhc-dto-java" to "0.3.0",
+    "hapi-fhir" to "6.10.5",
+    "httpclient5" to "5.2.3",
+    "mockito-kotlin" to "5.3.1",
     // Webjars
     "echarts" to "5.4.3",
-    "htmx.org" to "1.9.10"
+    "htmx.org" to "1.9.12"
 )
 
 java {
@@ -70,7 +70,7 @@ dependencies {
     implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}")
     implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}")
     implementation("com.jayway.jsonpath:json-path")
-    implementation("org.webjars:webjars-locator:0.50")
+    implementation("org.webjars:webjars-locator:0.52")
     implementation("org.webjars.npm:echarts:${versions["echarts"]}")
     implementation("org.webjars.npm:htmx.org:${versions["htmx.org"]}")
     runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
@@ -85,6 +85,8 @@ dependencies {
     testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}")
     integrationTestImplementation("org.testcontainers:junit-jupiter")
     integrationTestImplementation("org.testcontainers:postgresql")
+    // Override dependency version from org.testcontainers:junit-jupiter - CVE-2024-26308, CVE-2024-25710
+    integrationTestImplementation("org.apache.commons:commons-compress:1.26.1")
 }
 
 tasks.withType<KotlinCompile> {
@@ -111,7 +113,7 @@ task<Test>("integrationTest") {
 }
 
 tasks.named<BootBuildImage>("bootBuildImage") {
-    imageName.set("ghcr.io/ccc-mf/etl-processor")
+    imageName.set("ghcr.io/pcvolkmer/etl-processor")
 
     // Binding for CA Certs
     bindings.set(listOf(
@@ -121,7 +123,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
     environment.set(environment.get() + mapOf(
         // Enable this line to embed CA Certs into image on build time
         //"BP_EMBED_CERTS" to "true",
-        "BP_OCI_SOURCE" to "https://github.com/CCC-MF/etl-processor",
+        "BP_OCI_SOURCE" to "https://github.com/pcvolkmer/etl-processor",
         "BP_OCI_LICENSES" to "AGPLv3",
         "BP_OCI_DESCRIPTION" to "ETL Processor for bwHC MTB files"
     ))

src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt

@@ -57,6 +57,7 @@ class ReportService(
     data class Issue(val severity: Severity, val message: String)
 
     enum class Severity(@JsonValue val value: String) {
+        FATAL("fatal"),
         ERROR("error"),
         WARNING("warning"),
         INFO("info")

src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt

@@ -1,7 +1,7 @@
 /*
  * This file is part of ETL-Processor
  *
- * Copyright (c) 2023  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
+ * Copyright (c) 2024  Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published
@@ -31,31 +31,31 @@ import org.apache.commons.codec.digest.DigestUtils
 infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
     val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id)
 
-    this.episode.patient = patientPseudonym
-    this.carePlans.forEach { it.patient = patientPseudonym }
+    this.episode?.patient = patientPseudonym
+    this.carePlans?.forEach { it.patient = patientPseudonym }
     this.patient.id = patientPseudonym
-    this.claims.forEach { it.patient = patientPseudonym }
-    this.consent.patient = patientPseudonym
-    this.claimResponses.forEach { it.patient = patientPseudonym }
-    this.diagnoses.forEach { it.patient = patientPseudonym }
-    this.ecogStatus.forEach { it.patient = patientPseudonym }
-    this.familyMemberDiagnoses.forEach { it.patient = patientPseudonym }
-    this.geneticCounsellingRequests.forEach { it.patient = patientPseudonym }
-    this.histologyReevaluationRequests.forEach { it.patient = patientPseudonym }
-    this.histologyReports.forEach {
+    this.claims?.forEach { it.patient = patientPseudonym }
+    this.consent?.patient = patientPseudonym
+    this.claimResponses?.forEach { it.patient = patientPseudonym }
+    this.diagnoses?.forEach { it.patient = patientPseudonym }
+    this.ecogStatus?.forEach { it.patient = patientPseudonym }
+    this.familyMemberDiagnoses?.forEach { it.patient = patientPseudonym }
+    this.geneticCounsellingRequests?.forEach { it.patient = patientPseudonym }
+    this.histologyReevaluationRequests?.forEach { it.patient = patientPseudonym }
+    this.histologyReports?.forEach {
         it.patient = patientPseudonym
-        it.tumorMorphology.patient = patientPseudonym
+        it.tumorMorphology?.patient = patientPseudonym
     }
-    this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym }
-    this.molecularPathologyFindings.forEach { it.patient = patientPseudonym }
-    this.molecularTherapies.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
-    this.ngsReports.forEach { it.patient = patientPseudonym }
-    this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym }
-    this.rebiopsyRequests.forEach { it.patient = patientPseudonym }
-    this.recommendations.forEach { it.patient = patientPseudonym }
-    this.responses.forEach { it.patient = patientPseudonym }
-    this.studyInclusionRequests.forEach { it.patient = patientPseudonym }
-    this.specimens.forEach { it.patient = patientPseudonym }
+    this.lastGuidelineTherapies?.forEach { it.patient = patientPseudonym }
+    this.molecularPathologyFindings?.forEach { it.patient = patientPseudonym }
+    this.molecularTherapies?.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
+    this.ngsReports?.forEach { it.patient = patientPseudonym }
+    this.previousGuidelineTherapies?.forEach { it.patient = patientPseudonym }
+    this.rebiopsyRequests?.forEach { it.patient = patientPseudonym }
+    this.recommendations?.forEach { it.patient = patientPseudonym }
+    this.responses?.forEach { it.patient = patientPseudonym }
+    this.studyInclusionRequests?.forEach { it.patient = patientPseudonym }
+    this.specimens?.forEach { it.patient = patientPseudonym }
 }
 
 /**
@@ -73,151 +73,153 @@ infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService)
         return "$prefix$hash"
     }
 
-    this.episode.apply {
-        id = anonymize(id)
-    }
-    this.carePlans.onEach { carePlan ->
-        carePlan.apply {
-            id = anonymize(id)
-            diagnosis = anonymize(diagnosis)
-            geneticCounsellingRequest = anonymize(geneticCounsellingRequest)
-            rebiopsyRequests = rebiopsyRequests.map { anonymize(it) }
-            recommendations = recommendations.map { anonymize(it) }
-            studyInclusionRequests = studyInclusionRequests.map { anonymize(it) }
+    this.episode?.apply {
+        id = id?.let {
+            anonymize(it)
         }
     }
-    this.claims.onEach { claim ->
-        claim.apply {
-            id = anonymize(id)
-            therapy = anonymize(therapy)
+    this.carePlans?.onEach { carePlan ->
+        carePlan?.apply {
+            id = id?.let { anonymize(it) }
+            diagnosis = diagnosis?.let { anonymize(it) }
+            geneticCounsellingRequest = geneticCounsellingRequest?.let { anonymize(it) }
+            rebiopsyRequests = rebiopsyRequests.map { it?.let { anonymize(it) } }
+            recommendations = recommendations.map { it?.let { anonymize(it) } }
+            studyInclusionRequests = studyInclusionRequests.map { it?.let { anonymize(it) } }
         }
     }
-    this.claimResponses.onEach { claimResponse ->
-        claimResponse.apply {
-            id = anonymize(id)
-            claim = anonymize(claim)
+    this.claims?.onEach { claim ->
+        claim?.apply {
+            id = id?.let { anonymize(it) }
+            therapy = therapy?.let { anonymize(it) }
         }
     }
-    this.consent.apply {
-        id = anonymize(id)
-    }
-    this.diagnoses.onEach { diagnosis ->
-        diagnosis.apply {
-            id = anonymize(id)
-            histologyResults = histologyResults.map { anonymize(it) }
+    this.claimResponses?.onEach { claimResponse ->
+        claimResponse?.apply {
+            id = id?.let { anonymize(it) }
+            claim = claim?.let { anonymize(it) }
         }
     }
-    this.ecogStatus.onEach { ecogStatus ->
-        ecogStatus.apply {
-            id = anonymize(id)
+    this.consent?.apply {
+        id = id?.let { anonymize(it) }
+    }
+    this.diagnoses?.onEach { diagnosis ->
+        diagnosis?.apply {
+            id = id?.let { anonymize(it) }
+            histologyResults = histologyResults?.map { it?.let { anonymize(it) } }
         }
     }
-    this.familyMemberDiagnoses.onEach { familyMemberDiagnosis ->
-        familyMemberDiagnosis.apply {
-            id = anonymize(id)
+    this.ecogStatus?.onEach { ecogStatus ->
+        ecogStatus?.apply {
+            id = id?.let { anonymize(it) }
         }
     }
-    this.geneticCounsellingRequests.onEach { geneticCounsellingRequest -> 
-        geneticCounsellingRequest.apply { 
-            id = anonymize(id)
+    this.familyMemberDiagnoses?.onEach { familyMemberDiagnosis ->
+        familyMemberDiagnosis?.apply {
+            id = id?.let { anonymize(it) }
         }
     }
-    this.histologyReevaluationRequests.onEach { histologyReevaluationRequest ->
-        histologyReevaluationRequest.apply {
-            id = anonymize(id)
-            specimen = anonymize(specimen)
+    this.geneticCounsellingRequests?.onEach { geneticCounsellingRequest -> 
+        geneticCounsellingRequest?.apply { 
+            id = id?.let { anonymize(it) }
         }
     }
-    this.histologyReports.onEach { histologyReport ->
-        histologyReport.apply {
-            id = anonymize(id)
-            specimen = anonymize(specimen)
-            tumorMorphology.apply {
-                id = anonymize(id)
-                specimen = anonymize(specimen)
+    this.histologyReevaluationRequests?.onEach { histologyReevaluationRequest ->
+        histologyReevaluationRequest?.apply {
+            id = id?.let { anonymize(it) }
+            specimen = specimen?.let { anonymize(it) }
+        }
+    }
+    this.histologyReports?.onEach { histologyReport ->
+        histologyReport?.apply {
+            id = id?.let { anonymize(it) }
+            specimen = specimen?.let { anonymize(it) }
+            tumorMorphology?.apply {
+                id = id?.let { anonymize(it) }
+                specimen = specimen?.let { anonymize(it) }
             }
-            tumorCellContent.apply {
-                id = anonymize(id)
-                specimen = anonymize(specimen)
+            tumorCellContent?.apply {
+                id = id?.let { anonymize(it) }
+                specimen = specimen?.let { anonymize(it) }
             }
         }
     }
-    this.lastGuidelineTherapies.onEach { lastGuidelineTherapy ->
-        lastGuidelineTherapy.apply {
-            id = anonymize(id)
-            diagnosis = anonymize(diagnosis)
+    this.lastGuidelineTherapies?.onEach { lastGuidelineTherapy ->
+        lastGuidelineTherapy?.apply {
+            id = id?.let { anonymize(it) }
+            diagnosis = diagnosis?.let { anonymize(it) }
         }
     }
-    this.molecularPathologyFindings.onEach { molecularPathologyFinding ->
-        molecularPathologyFinding.apply {
-            id = anonymize(id)
-            specimen = anonymize(specimen)
+    this.molecularPathologyFindings?.onEach { molecularPathologyFinding ->
+        molecularPathologyFinding?.apply {
+            id = id?.let { anonymize(it) }
+            specimen = specimen?.let { anonymize(it) }
         }
     }
-    this.molecularTherapies.onEach { molecularTherapy ->
-        molecularTherapy.apply {
-            history.onEach { history ->
-                history.apply {
-                    id = anonymize(id)
-                    basedOn = anonymize(basedOn)
+    this.molecularTherapies?.onEach { molecularTherapy ->
+        molecularTherapy?.apply {
+            history?.onEach { history ->
+                history?.apply {
+                    id = id?.let { anonymize(it) }
+                    basedOn = basedOn?.let { anonymize(it) }
                 }
             }
         }
     }
-    this.ngsReports.onEach { ngsReport ->
-        ngsReport.apply {
-            id = anonymize(id)
-            specimen = anonymize(specimen)
-            tumorCellContent.apply {
-                id = anonymize(id)
-                specimen = anonymize(specimen)
+    this.ngsReports?.onEach { ngsReport ->
+        ngsReport?.apply {
+            id = id?.let { anonymize(it) }
+            specimen = specimen?.let { anonymize(it) }
+            tumorCellContent?.apply {
+                id = id?.let { anonymize(it) }
+                specimen = specimen?.let { anonymize(it) }
             }
-            simpleVariants.onEach { simpleVariant ->
-                simpleVariant.apply {
-                    id = anonymize(id)
+            simpleVariants?.onEach { simpleVariant ->
+                simpleVariant?.apply {
+                    id = id?.let { anonymize(it) }
                 }
             }
         }
     }
-    this.previousGuidelineTherapies.onEach { previousGuidelineTherapy ->
-        previousGuidelineTherapy.apply {
-            id = anonymize(id)
-            diagnosis = anonymize(diagnosis)
-            this.medication.forEach { medication ->
-                medication.apply {
-                    id = anonymize(id)
+    this.previousGuidelineTherapies?.onEach { previousGuidelineTherapy ->
+        previousGuidelineTherapy?.apply {
+            id = id?.let { anonymize(it) }
+            diagnosis = diagnosis?.let { anonymize(it) }
+            medication.forEach { medication ->
+                medication?.apply {
+                    id = id?.let { anonymize(it) }
                 }
             }
         }
     }
-    this.rebiopsyRequests.onEach { rebiopsyRequest ->
-        rebiopsyRequest.apply {
-            id = anonymize(id)
-            specimen = anonymize(specimen)
+    this.rebiopsyRequests?.onEach { rebiopsyRequest ->
+        rebiopsyRequest?.apply {
+            id = id?.let { anonymize(it) }
+            specimen = specimen?.let { anonymize(it) }
         }
     }
-    this.recommendations.onEach { recommendation ->
-        recommendation.apply {
-            id = anonymize(id)
-            diagnosis = anonymize(diagnosis)
-            ngsReport = anonymize(ngsReport)
+    this.recommendations?.onEach { recommendation ->
+        recommendation?.apply {
+            id = id?.let { anonymize(it) }
+            diagnosis = diagnosis?.let { anonymize(it) }
+            ngsReport = ngsReport?.let { anonymize(it) }
         }
     }
-    this.responses.onEach { response ->
-        response.apply {
-            id = anonymize(id)
-            therapy = anonymize(therapy)
+    this.responses?.onEach { response ->
+        response?.apply {
+            id = id?.let { anonymize(it) }
+            therapy = therapy?.let { anonymize(it) }
         }
     }
-    this.studyInclusionRequests.onEach { studyInclusionRequest ->
-        studyInclusionRequest.apply {
-            id = anonymize(id)
-            reason = anonymize(reason)
+    this.studyInclusionRequests?.onEach { studyInclusionRequest ->
+        studyInclusionRequest?.apply {
+            id = id?.let { anonymize(it) }
+            reason = reason?.let { anonymize(it) }
         }
     }
-    this.specimens.onEach { specimen ->
-        specimen.apply {
-            id = anonymize(id)
+    this.specimens?.onEach { specimen ->
+        specimen?.apply {
+            id = id?.let { anonymize(it) }
         }
     }
 }

src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt

@@ -134,7 +134,6 @@ class StatisticsRestController(
     @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
     fun updater(): Flux<ServerSentEvent<Any>> {
         return statisticsUpdateProducer.asFlux().flatMap {
-            println(it)
             Flux.fromIterable(
                 listOf(
                     ServerSentEvent.builder<Any>()

src/main/resources/templates/report.html

@@ -55,6 +55,7 @@
                 <td th:if="${issue.severity.value == 'info'}" class="bg-blue"><small>[[ ${issue.severity} ]]</small></td>
                 <td th:if="${issue.severity.value == 'warning'}" class="bg-yellow"><small>[[ ${issue.severity} ]]</small></td>
                 <td th:if="${issue.severity.value == 'error'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td>
+                <td th:if="${issue.severity.value == 'fatal'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td>
                 <td>[[ ${issue.message} ]]</td>
             </tr>
             </tbody>

src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt

@@ -134,4 +134,65 @@ class ExtensionsTest {
            .isEqualTo("TESTDOMAIN44e20a53bbbf9f3ae39626d05df7014dcd77d6098")
     }
 
+    @Test
+    fun shouldNotThrowExceptionOnNullValues(@Mock pseudonymizeService: PseudonymizeService) {
+        doAnswer {
+            it.arguments[0]
+            "PSEUDO-ID"
+        }.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
+
+        doAnswer {
+            "TESTDOMAIN"
+        }.whenever(pseudonymizeService).prefix()
+
+        val mtbFile = MtbFile.builder()
+            .withPatient(
+                Patient.builder()
+                    .withId("1")
+                    .withBirthDate("2000-08-08")
+                    .withGender(Patient.Gender.MALE)
+                    .build()
+            )
+            .withConsent(
+                Consent.builder()
+                    .withId("1")
+                    .withStatus(Consent.Status.ACTIVE)
+                    .withPatient("123")
+                    .build()
+            )
+            .withEpisode(
+                Episode.builder()
+                    .withId("1")
+                    .withPatient("1")
+                    .withPeriod(PeriodStart("2023-08-08"))
+                    .build()
+            )
+            .withClaims(null)
+            .withDiagnoses(null)
+            .withCarePlans(null)
+            .withClaimResponses(null)
+            .withEcogStatus(null)
+            .withFamilyMemberDiagnoses(null)
+            .withGeneticCounsellingRequests(null)
+            .withHistologyReevaluationRequests(null)
+            .withHistologyReports(null)
+            .withLastGuidelineTherapies(null)
+            .withMolecularPathologyFindings(null)
+            .withMolecularTherapies(null)
+            .withNgsReports(null)
+            .withPreviousGuidelineTherapies(null)
+            .withRebiopsyRequests(null)
+            .withRecommendations(null)
+            .withResponses(null)
+            .withStudyInclusionRequests(null)
+            .withSpecimens(null)
+            .build()
+
+        mtbFile.pseudonymizeWith(pseudonymizeService)
+        mtbFile.anonymizeContentWith(pseudonymizeService)
+
+
+        assertThat(mtbFile.episode.id).isNotNull()
+    }
+
 }

src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt

@@ -43,20 +43,23 @@ class ReportServiceTest {
                 "issues": [
                     { "severity": "info", "message": "Info Message" },
                     { "severity": "warning", "message": "Warning Message" },
-                    { "severity": "error", "message": "Error Message" }
+                    { "severity": "error", "message": "Error Message" },
+                    { "severity": "fatal", "message": "Fatal Message" }
                 ]
             }
         """.trimIndent()
 
         val actual = this.reportService.deserialize(json)
 
-        assertThat(actual).hasSize(3)
-        assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR)
-        assertThat(actual[0].message).isEqualTo("Error Message")
-        assertThat(actual[1].severity).isEqualTo(ReportService.Severity.WARNING)
-        assertThat(actual[1].message).isEqualTo("Warning Message")
-        assertThat(actual[2].severity).isEqualTo(ReportService.Severity.INFO)
-        assertThat(actual[2].message).isEqualTo("Info Message")
+        assertThat(actual).hasSize(4)
+        assertThat(actual[0].severity).isEqualTo(ReportService.Severity.FATAL)
+        assertThat(actual[0].message).isEqualTo("Fatal Message")
+        assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR)
+        assertThat(actual[1].message).isEqualTo("Error Message")
+        assertThat(actual[2].severity).isEqualTo(ReportService.Severity.WARNING)
+        assertThat(actual[2].message).isEqualTo("Warning Message")
+        assertThat(actual[3].severity).isEqualTo(ReportService.Severity.INFO)
+        assertThat(actual[3].message).isEqualTo("Info Message")
     }
 
     @Test