Limit the log size so that dashboard charts are not overloaded

Added configuration constants
This script is designed to be ran by Dash (which is faster than Bash)
This commit is contained in:
Diogo Cordeiro 2018-10-03 03:10:49 +01:00
parent 1310b92a33
commit 838cc1d950
2 changed files with 21 additions and 13 deletions

View File

@ -46,7 +46,6 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.2/Chart.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-annotation/0.5.7/chartjs-plugin-annotation.min.js"></script>
<script src="chart.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.1.1/socket.io.slim.js"></script>
<script src="ingestion-rate.js"></script>
<footer id="footer">
<p><a href="https://github.com/diogogithub/spark-art-dashboard">ART Dashboard</a> and <a href="https://github.com/diogogithub/excalibur_template/">Excalibur template</a> by <a href="https://www.diogo.site/">Diogo Cordeiro</a> - <a href="https://www.inesc-id.pt/">INESC-ID</a>

View File

@ -1,17 +1,26 @@
#!/bin/bash
#!/bin/sh
# last stats file to read from
# CSV file name
FILENAME=$(ls stats-* | tail -n 1)
# number of lines to read
# chunk file name
CHUNK_FILENAME="current-log-provider-chunk.csv"
# number of log lines for chunk
N=100
# chunk filename
CHUNK_FILENAME="log-provider-chunk.csv"
# server port
PORT=4870
header=$(head -n 1 $FILENAME)
# CSV header
header=$(head -n 1 "$FILENAME")
while true; do
paste <(echo $header) <(tail -n +2 $FILENAME | tail -n $N) --d '' > $CHUNK_FILENAME
{ echo -ne "HTTP/1.0 200 OK\r\nContent-Length: $(wc -c $CHUNK_FILENAME)\r\nAccess-Control-Allow-Origin: *\r\n\r"; cat $CHUNK_FILENAME; } | nc -v -l -p 4870
# We don't want repeated headers
if [ $(wc -l < "$FILENAME") -gt "$N" ]; then
{ echo "$header"; tail -n "$N" $FILENAME; } > "$CHUNK_FILENAME"
else
cp -f -- "$FILENAME" "$CHUNK_FILENAME"
fi
{ echo -ne "HTTP/1.0 200 OK\r\nContent-Length: $(wc -c <"current-log-provider-chunk.csv")\r\nAccess-Control-Allow-Origin: *\r\n\r"; cat "current-log-provider-chunk.csv"; } | nc -v -l -p "$PORT"
done