From 1a2d4ea7a20cddd61a89f11ed3d450a0381df6ab Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 25 Jul 2023 20:55:32 +0200 Subject: [PATCH] (Near) realtime update of statistics charts --- build.gradle.kts | 2 + .../etl/processor/config/AppConfiguration.kt | 9 + .../etl/processor/web/MtbFileController.kt | 7 +- .../etl/processor/web/StatisticsController.kt | 5 +- .../processor/web/StatisticsRestController.kt | 48 +++- src/main/resources/static/scripts.js | 227 +++++++++++------- src/main/resources/templates/statistics.html | 14 +- 7 files changed, 208 insertions(+), 104 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0cbf77e..3371bb4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation("org.springframework.kafka:spring-kafka") implementation("org.flywaydb:flyway-mysql") implementation("commons-codec:commons-codec") + implementation("io.projectreactor.kotlin:reactor-kotlin-extensions") implementation("de.ukw.ccc:bwhc-dto-java:0.2.0") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") runtimeOnly("org.postgresql:postgresql") @@ -45,6 +46,7 @@ dependencies { annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("io.projectreactor:reactor-test") } tasks.withType { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index ed3be5d..e4a97ee 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -27,12 +27,16 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import org.reactivestreams.Publisher import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.kafka.core.KafkaTemplate +import reactor.core.publisher.Flux +import reactor.core.publisher.Sinks import java.net.URI +import java.time.Duration @Configuration @EnableConfigurationProperties( @@ -78,5 +82,10 @@ class AppConfiguration { return KafkaMtbFileSender(kafkaTemplate, objectMapper) } + @Bean + fun statisticsUpdateProducer(): Sinks.Many { + return Sinks.many().multicast().directBestEffort() + } + } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt index 835f3de..9cbb52a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt @@ -34,13 +34,15 @@ import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RestController +import reactor.core.publisher.Sinks @RestController class MtbFileController( private val pseudonymizeService: PseudonymizeService, private val senders: List, private val requestRepository: RequestRepository, - private val objectMapper: ObjectMapper + private val objectMapper: ObjectMapper, + private val statisticsUpdateProducer: Sinks.Many ) { private val logger = LoggerFactory.getLogger(MtbFileController::class.java) @@ -63,6 +65,7 @@ class MtbFileController( report = Report("Duplikat erkannt - keine Daten weitergeleitet") ) ) + statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) return ResponseEntity.noContent().build() } @@ -110,6 +113,8 @@ class MtbFileController( ) ) + statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) + return if (requestStatus == RequestStatus.ERROR) { ResponseEntity.unprocessableEntity().build() } else { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt index 05b84ff..adc1e2b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt @@ -20,15 +20,18 @@ package dev.dnpm.etl.processor.web import org.springframework.stereotype.Controller +import org.springframework.ui.Model import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping +import java.time.Instant @Controller @RequestMapping(path = ["/statistics"]) class StatisticsController { @GetMapping - fun index(): String { + fun index(model: Model): String { + model.addAttribute("now", Instant.now()) return "statistics" } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt index 8d5cb0e..2741fd3 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt @@ -21,9 +21,15 @@ package dev.dnpm.etl.processor.web import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus +import org.reactivestreams.Publisher +import org.springframework.http.MediaType +import org.springframework.http.codec.ServerSentEvent import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController +import reactor.core.publisher.Flux +import reactor.core.publisher.Sinks +import reactor.kotlin.core.publisher.toFlux import java.time.Instant import java.time.Month import java.time.ZoneId @@ -34,6 +40,7 @@ import java.time.temporal.TemporalUnit @RestController @RequestMapping(path = ["/statistics"]) class StatisticsRestController( + private val statisticsUpdateProducer: Sinks.Many, private val requestRepository: RequestRepository ) { @@ -68,13 +75,15 @@ class StatisticsRestController( .toMap() Pair( it.key.toString(), - DateNameValues(it.key.toString(), NameValues( - error = requestList[RequestStatus.ERROR] ?: 0, - warning = requestList[RequestStatus.WARNING] ?: 0, - success = requestList[RequestStatus.SUCCESS] ?: 0, - duplication = requestList[RequestStatus.DUPLICATION] ?: 0, - unknown = requestList[RequestStatus.UNKNOWN] ?: 0, - )) + DateNameValues( + it.key.toString(), NameValues( + error = requestList[RequestStatus.ERROR] ?: 0, + warning = requestList[RequestStatus.WARNING] ?: 0, + success = requestList[RequestStatus.SUCCESS] ?: 0, + duplication = requestList[RequestStatus.DUPLICATION] ?: 0, + unknown = requestList[RequestStatus.UNKNOWN] ?: 0, + ) + ) ) }.toMap() @@ -86,10 +95,33 @@ class StatisticsRestController( .sortedBy { it.date } } + @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE]) + fun updater(): Flux> { + return statisticsUpdateProducer.asFlux().flatMap { + Flux.fromIterable( + listOf( + ServerSentEvent.builder() + .event("requeststates").id("none").data(this.requestStates()) + .build(), + ServerSentEvent.builder() + .event("requestslastmonth").id("none").data(this.requestsLastMonth()) + .build() + ) + ) + + } + } + } data class NameValue(val name: String, val value: Int, val color: String) data class DateNameValues(val date: String, val nameValues: NameValues) -data class NameValues(val error: Int = 0, val warning: Int = 0, val success: Int = 0, val duplication: Int = 0, val unknown: Int = 0) \ No newline at end of file +data class NameValues( + val error: Int = 0, + val warning: Int = 0, + val success: Int = 0, + val duplication: Int = 0, + val unknown: Int = 0 +) \ No newline at end of file diff --git a/src/main/resources/static/scripts.js b/src/main/resources/static/scripts.js index 8f08ee5..2285167 100644 --- a/src/main/resources/static/scripts.js +++ b/src/main/resources/static/scripts.js @@ -13,101 +13,144 @@ window.onload = () => { }); }; -function drawPieChart(url, elemId, title) { - fetch(url) - .then(resp => resp.json()) - .then(data => { - let chartDom = document.getElementById(elemId); - let chart = echarts.init(chartDom); - let option= { - title: { - text: title, - left: 'center' - }, - tooltip: { - trigger: 'item' - }, - color: data.map(i => i.color), - series: [ - { - type: 'pie', - radius: ['40%', '70%'], - avoidLabelOverlap: false, - label: { - show: false, - position: 'center' - }, - labelLine: { - show: false - }, - data: data - } - ] - }; +function drawPieChart(url, elemId, title, data) { + if (data) { + update(elemId, data); + } else { + fetch(url) + .then(resp => resp.json()) + .then(d => { + draw(elemId, title, d); + update(elemId, d); + }); + } - option && chart.setOption(option); - }); + function update(elemId, data) { + let chartDom = document.getElementById(elemId); + let chart = echarts.init(chartDom); + let option = { + color: data.map(i => i.color), + animationDuration: 250, + animationDurationUpdate: 250, + series: [ + { + type: 'pie', + radius: ['40%', '70%'], + avoidLabelOverlap: false, + label: { + show: false, + position: 'center' + }, + labelLine: { + show: false + }, + data: data + } + ] + }; + + option && chart.setOption(option); + } + + function draw(elemId, title, data) { + let chartDom = document.getElementById(elemId); + let chart = echarts.init(chartDom); + let option= { + title: { + text: title, + left: 'center' + }, + tooltip: { + trigger: 'item' + }, + color: data.map(i => i.color), + animationDuration: 250, + animationDurationUpdate: 250 + }; + + option && chart.setOption(option); + } } -function drawBarChart(url, elemId, title) { - fetch(url) - .then(resp => resp.json()) - .then(data => { - let chartDom = document.getElementById(elemId); - let chart = echarts.init(chartDom); - let option= { - title: { - text: title, - left: 'center' - }, - xAxis: { - type: 'category', - data: data.map(i => dateFormat.format(Date.parse(i.date))) - }, - yAxis: { - type: 'value', - minInterval: 2, - }, - tooltip: { - trigger: 'item' - }, - animation: false, - color: ['slategray', 'red', 'darkorange', 'green', 'slategray'], - series: [ - { - name: 'UNKNOWN', - type: 'bar', - stack: 'total', - data: data.map(i => i.nameValues.unknown) - }, - { - name: 'ERROR', - type: 'bar', - stack: 'total', - data: data.map(i => i.nameValues.error) - }, - { - name: 'WARNING', - type: 'bar', - stack: 'total', - data: data.map(i => i.nameValues.warning) - }, - { - name: 'SUCCESS', - type: 'bar', - stack: 'total', - data: data.map(i => i.nameValues.success) - }, - { - name: 'DUPLICATION', - type: 'bar', - stack: 'total', - data: data.map(i => i.nameValues.duplication) - } - ] - }; +function drawBarChart(url, elemId, title, data) { + if (data) { + update(elemId, data); + } else { + fetch(url) + .then(resp => resp.json()) + .then(data => { + draw(elemId, title, data); + update(elemId, data); + }); + } - option && chart.setOption(option); - }); + function update(elemId, data) { + let chartDom = document.getElementById(elemId); + let chart = echarts.init(chartDom); + + let option = { + series: [ + { + name: 'UNKNOWN', + type: 'bar', + stack: 'total', + data: data.map(i => i.nameValues.unknown) + }, + { + name: 'ERROR', + type: 'bar', + stack: 'total', + data: data.map(i => i.nameValues.error) + }, + { + name: 'WARNING', + type: 'bar', + stack: 'total', + data: data.map(i => i.nameValues.warning) + }, + { + name: 'SUCCESS', + type: 'bar', + stack: 'total', + data: data.map(i => i.nameValues.success) + }, + { + name: 'DUPLICATION', + type: 'bar', + stack: 'total', + data: data.map(i => i.nameValues.duplication) + } + ] + }; + + option && chart.setOption(option); + } + + function draw(elemId, title, data) { + let chartDom = document.getElementById(elemId); + let chart = echarts.init(chartDom); + let option= { + title: { + text: title, + left: 'center' + }, + xAxis: { + type: 'category', + data: data.map(i => dateFormat.format(Date.parse(i.date))) + }, + yAxis: { + type: 'value', + minInterval: 1 + }, + tooltip: { + trigger: 'item' + }, + color: ['slategray', 'red', 'darkorange', 'green', 'slategray'], + animationDuration: 250, + animationDurationUpdate: 250 + }; + + option && chart.setOption(option); + } } \ No newline at end of file diff --git a/src/main/resources/templates/statistics.html b/src/main/resources/templates/statistics.html index 3f2a11b..007303e 100644 --- a/src/main/resources/templates/statistics.html +++ b/src/main/resources/templates/statistics.html @@ -17,8 +17,18 @@ \ No newline at end of file