(Near) realtime update of statistics charts

build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
     implementation("org.springframework.kafka:spring-kafka")
     implementation("org.flywaydb:flyway-mysql")
     implementation("commons-codec:commons-codec")
+    implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")
     implementation("de.ukw.ccc:bwhc-dto-java:0.2.0")
     runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
     runtimeOnly("org.postgresql:postgresql")
@@ -45,6 +46,7 @@ dependencies {
     annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
     providedRuntime("org.springframework.boot:spring-boot-starter-tomcat")
     testImplementation("org.springframework.boot:spring-boot-starter-test")
+    testImplementation("io.projectreactor:reactor-test")
 }
 
 tasks.withType<KotlinCompile> {

src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt

@@ -27,12 +27,16 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
 import dev.dnpm.etl.processor.pseudonym.Generator
 import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
 import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
+import org.reactivestreams.Publisher
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.kafka.core.KafkaTemplate
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
 import java.net.URI
+import java.time.Duration
 
 @Configuration
 @EnableConfigurationProperties(
@@ -78,5 +82,10 @@ class AppConfiguration {
         return KafkaMtbFileSender(kafkaTemplate, objectMapper)
     }
 
+    @Bean
+    fun statisticsUpdateProducer(): Sinks.Many<Any> {
+        return Sinks.many().multicast().directBestEffort()
+    }
+
 }
 

src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt

@@ -34,13 +34,15 @@ import org.springframework.http.ResponseEntity
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RestController
+import reactor.core.publisher.Sinks
 
 @RestController
 class MtbFileController(
     private val pseudonymizeService: PseudonymizeService,
     private val senders: List<MtbFileSender>,
     private val requestRepository: RequestRepository,
-    private val objectMapper: ObjectMapper
+    private val objectMapper: ObjectMapper,
+    private val statisticsUpdateProducer: Sinks.Many<Any>
 ) {
 
     private val logger = LoggerFactory.getLogger(MtbFileController::class.java)
@@ -63,6 +65,7 @@ class MtbFileController(
                     report = Report("Duplikat erkannt - keine Daten weitergeleitet")
                 )
             )
+            statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
             return ResponseEntity.noContent().build()
         }
 
@@ -110,6 +113,8 @@ class MtbFileController(
             )
         )
 
+        statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
+
         return if (requestStatus == RequestStatus.ERROR) {
             ResponseEntity.unprocessableEntity().build()
         } else {

src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt

@@ -20,15 +20,18 @@
 package dev.dnpm.etl.processor.web
 
 import org.springframework.stereotype.Controller
+import org.springframework.ui.Model
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
+import java.time.Instant
 
 @Controller
 @RequestMapping(path = ["/statistics"])
 class StatisticsController {
 
     @GetMapping
-    fun index(): String {
+    fun index(model: Model): String {
+        model.addAttribute("now", Instant.now())
         return "statistics"
     }
 

src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt

@@ -21,9 +21,15 @@ package dev.dnpm.etl.processor.web
 
 import dev.dnpm.etl.processor.monitoring.RequestRepository
 import dev.dnpm.etl.processor.monitoring.RequestStatus
+import org.reactivestreams.Publisher
+import org.springframework.http.MediaType
+import org.springframework.http.codec.ServerSentEvent
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RestController
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
+import reactor.kotlin.core.publisher.toFlux
 import java.time.Instant
 import java.time.Month
 import java.time.ZoneId
@@ -34,6 +40,7 @@ import java.time.temporal.TemporalUnit
 @RestController
 @RequestMapping(path = ["/statistics"])
 class StatisticsRestController(
+    private val statisticsUpdateProducer: Sinks.Many<Any>,
     private val requestRepository: RequestRepository
 ) {
 
@@ -68,13 +75,15 @@ class StatisticsRestController(
                     .toMap()
                 Pair(
                     it.key.toString(),
-                    DateNameValues(it.key.toString(), NameValues(
+                    DateNameValues(
+                        it.key.toString(), NameValues(
                             error = requestList[RequestStatus.ERROR] ?: 0,
                             warning = requestList[RequestStatus.WARNING] ?: 0,
                             success = requestList[RequestStatus.SUCCESS] ?: 0,
                             duplication = requestList[RequestStatus.DUPLICATION] ?: 0,
                             unknown = requestList[RequestStatus.UNKNOWN] ?: 0,
-                    ))
+                        )
+                    )
                 )
             }.toMap()
 
@@ -86,10 +95,33 @@ class StatisticsRestController(
             .sortedBy { it.date }
     }
 
+    @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
+    fun updater(): Flux<ServerSentEvent<Any>> {
+        return statisticsUpdateProducer.asFlux().flatMap {
+            Flux.fromIterable(
+                listOf(
+                    ServerSentEvent.builder<Any>()
+                        .event("requeststates").id("none").data(this.requestStates())
+                        .build(),
+                    ServerSentEvent.builder<Any>()
+                        .event("requestslastmonth").id("none").data(this.requestsLastMonth())
+                        .build()
+                )
+            )
+
+        }
+    }
+
 }
 
 data class NameValue(val name: String, val value: Int, val color: String)
 
 data class DateNameValues(val date: String, val nameValues: NameValues)
 
-data class NameValues(val error: Int = 0, val warning: Int = 0, val success: Int = 0, val duplication: Int = 0, val unknown: Int = 0)
+data class NameValues(
+    val error: Int = 0,
+    val warning: Int = 0,
+    val success: Int = 0,
+    val duplication: Int = 0,
+    val unknown: Int = 0
+)

src/main/resources/static/scripts.js

@@ -13,21 +13,26 @@ window.onload = () => {
     });
 };
 
-function drawPieChart(url, elemId, title) {
+function drawPieChart(url, elemId, title, data) {
+    if (data) {
+        update(elemId, data);
+    } else {
         fetch(url)
             .then(resp => resp.json())
-        .then(data => {
+            .then(d => {
+                draw(elemId, title, d);
+                update(elemId, d);
+            });
+    }
+
+    function update(elemId, data) {
         let chartDom = document.getElementById(elemId);
         let chart = echarts.init(chartDom);
-            let option= {
+
-                title: {
+        let option = {
-                    text: title,
-                    left: 'center'
-                },
-                tooltip: {
-                    trigger: 'item'
-                },
             color: data.map(i => i.color),
+            animationDuration: 250,
+            animationDurationUpdate: 250,
             series: [
                 {
                     type: 'pie',
@@ -46,14 +51,9 @@ function drawPieChart(url, elemId, title) {
         };
 
         option && chart.setOption(option);
-        });
+    }
 
-}
+    function draw(elemId, title, data) {
-
-function drawBarChart(url, elemId, title) {
-    fetch(url)
-        .then(resp => resp.json())
-        .then(data => {
         let chartDom = document.getElementById(elemId);
         let chart = echarts.init(chartDom);
         let option= {
@@ -61,19 +61,35 @@ function drawBarChart(url, elemId, title) {
                 text: title,
                 left: 'center'
             },
-                xAxis: {
-                    type: 'category',
-                    data: data.map(i => dateFormat.format(Date.parse(i.date)))
-                },
-                yAxis: {
-                    type: 'value',
-                    minInterval: 2,
-                },
             tooltip: {
                 trigger: 'item'
             },
-                animation: false,
+            color: data.map(i => i.color),
-                color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
+            animationDuration: 250,
+            animationDurationUpdate: 250
+        };
+
+        option && chart.setOption(option);
+    }
+}
+
+function drawBarChart(url, elemId, title, data) {
+    if (data) {
+        update(elemId, data);
+    } else {
+        fetch(url)
+            .then(resp => resp.json())
+            .then(data => {
+                draw(elemId, title, data);
+                update(elemId, data);
+            });
+    }
+
+    function update(elemId, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+
+        let option = {
             series: [
                 {
                     name: 'UNKNOWN',
@@ -109,5 +125,32 @@ function drawBarChart(url, elemId, title) {
         };
 
         option && chart.setOption(option);
-        });
+    }
+
+    function draw(elemId, title, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+        let option= {
+            title: {
+                text: title,
+                left: 'center'
+            },
+            xAxis: {
+                type: 'category',
+                data: data.map(i => dateFormat.format(Date.parse(i.date)))
+            },
+            yAxis: {
+                type: 'value',
+                minInterval: 1
+            },
+            tooltip: {
+                trigger: 'item'
+            },
+            color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
+            animationDuration: 250,
+            animationDurationUpdate: 250
+        };
+
+        option && chart.setOption(option);
+    }
 }

src/main/resources/templates/statistics.html

@@ -17,8 +17,18 @@
     <script th:src="@{/echarts.min.js}"></script>
     <script th:src="@{/scripts.js}"></script>
     <script>
-        drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung der Anfragen');
+        window.onload = () => {
-        drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats');
+            drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen');
+            drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen der letzten 30 Tage');
+
+            const eventSource = new EventSource('statistics/events');
+            eventSource.addEventListener('requeststates', event => {
+                drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen', JSON.parse(event.data));
+            });
+            eventSource.addEventListener('requestslastmonth', event => {
+                drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats', JSON.parse(event.data));
+            });
+        }
     </script>
 </body>
 </html>