flucky/PKGBUILD
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

test(pkg/storage): add test for compression and rounding

Browse Source
This commit is contained in:
Markus Pesch
2019-10-11 12:36:00 +02:00
parent 86f598bc7d
commit 25dd99cd3d
68 changed files with 7713 additions and 1481 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/daemon/daemon.go
View File

@ -9,12 +9,12 @@ import (
"github.com/Masterminds/semver"
"github.com/go-flucky/flucky/pkg/config"
"github.com/volker-raschek/go-logger/pkg/logger"
"github.com/go-flucky/flucky/pkg/rgbled"
"github.com/go-flucky/flucky/pkg/sensor"
"github.com/go-flucky/flucky/pkg/storage/db"
"github.com/go-flucky/flucky/pkg/storage/logfile"
"github.com/go-flucky/flucky/pkg/types"
"github.com/volker-raschek/go-logger/pkg/logger"
)
var (
@ -102,7 +102,7 @@ func Start(cnf *config.Configuration, cleanCacheInterval time.Duration, compress
flogger.Error("Can not turn on blue info light: %v", err)
}
if err := logfile.Append(measuredValuesLogfile, compression, round, measuredValuesCache); err != nil {
if err := logfile.Append(measuredValuesLogfile, measuredValuesCache); err != nil {
err2 := rgbled.Error(rgbLEDs)
if err2 != nil {
flogger.Error("Can not turn on red info light: %v", err2)
@ -136,7 +136,7 @@ func Start(cnf *config.Configuration, cleanCacheInterval time.Duration, compress
}
flogger.Warn("Save remaining data from the cache")
err = logfile.Append(measuredValuesLogfile, compression, round, measuredValuesCache)
err = logfile.Append(measuredValuesLogfile, measuredValuesCache)
if err != nil {
flogger.Fatal("%v", err)
}

11
pkg/storage/logfile/csv.go
View File

@ -80,12 +80,15 @@ func (cl *csvLogfile) Read() ([]*types.MeasuredValue, error) {
}
// Creation date
creationDate, err := time.Parse(format.TimeFormat, record[6])
if err != nil {
return nil, fmt.Errorf("%v %v: %v", errorParseTime, record[6], err)
if record[6] != "null" {
creationDate, err := time.Parse(format.TimeFormat, record[6])
if err != nil {
return nil, fmt.Errorf("%v %v: %v", errorParseTime, record[6], err)
}
measuredValue.CreationDate = &creationDate
}
measuredValue.CreationDate = creationDate
// Update date
if record[7] != "null" {
updateDate, err := time.Parse(format.TimeFormat, record[7])
if err != nil {

75
pkg/storage/logfile/logfile.go
View File

@ -1,9 +1,7 @@
package logfile
import (
"math"
"path/filepath"
"sort"
"github.com/go-flucky/flucky/pkg/internal/format"
"github.com/go-flucky/flucky/pkg/types"
@ -12,13 +10,7 @@ import (
// var validUUID = regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
// Append adds an array of several measured values to a logfile
func Append(logfile Logfile, compression bool, round float64, measuredValues []*types.MeasuredValue) error {
if round != 0 {
for _, measuredValue := range measuredValues {
measuredValue.Value = math.Round(measuredValue.Value/round) * round
}
}
func Append(logfile Logfile, measuredValues []*types.MeasuredValue) error {
allMeasuredValues, err := logfile.Read()
if err != nil {
@ -27,10 +19,6 @@ func Append(logfile Logfile, compression bool, round float64, measuredValues []*
allMeasuredValues = append(allMeasuredValues, measuredValues...)
if compression {
allMeasuredValues = Compression(allMeasuredValues)
}
err = logfile.Write(allMeasuredValues)
if err != nil {
return err
@ -40,61 +28,6 @@ func Append(logfile Logfile, compression bool, round float64, measuredValues []*
return nil
}
// Compression the measured values. The system checks whether the measured values
// of the same type correspond to those of the predecessor. If this is the case,
// the current value is discarded and the validity date of the previous value is
// set to that of the current value. This means that no information is lost.
// Only the validity period of the measured value is increased.
func Compression(measuredValues []*types.MeasuredValue) []*types.MeasuredValue {
compressedMeasuredValues := make([]*types.MeasuredValue, 0)
lastMeasuredValuesBySensors := make(map[string]map[types.MeasuredValueType]*types.MeasuredValue, 0)
// Sort all measured values according to the start time of the validity date
// in order to successfully implement the subsequent compression.
sort.SliceStable(measuredValues, func(i int, j int) bool {
return measuredValues[i].FromDate.Before(measuredValues[j].TillDate)
})
now := format.FormatedTime()
for _, measuredValue := range measuredValues {
if _, ok := lastMeasuredValuesBySensors[measuredValue.SensorID]; !ok {
lastMeasuredValuesBySensors[measuredValue.SensorID] = make(map[types.MeasuredValueType]*types.MeasuredValue, 0)
}
if _, ok := lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType]; !ok {
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType] = measuredValue
continue
}
if lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].Value == measuredValue.Value {
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].TillDate = measuredValue.TillDate
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].UpdateDate = &now
} else if lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].Value != measuredValue.Value {
compressedMeasuredValues = append(compressedMeasuredValues, lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType])
delete(lastMeasuredValuesBySensors[measuredValue.SensorID], measuredValue.ValueType)
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType] = measuredValue
}
}
// Copy all remaining entries from the map into the cache array
for _, lastMeasuredValuesBySensor := range lastMeasuredValuesBySensors {
for _, measuredValueType := range types.MeasuredValueTypes {
if measuredValue, ok := lastMeasuredValuesBySensor[measuredValueType]; ok {
compressedMeasuredValues = append(compressedMeasuredValues, measuredValue)
}
}
}
// Sort all measured values again to include the measured values from the
// cache.
sort.SliceStable(compressedMeasuredValues, func(i int, j int) bool {
return compressedMeasuredValues[i].FromDate.Before(compressedMeasuredValues[j].FromDate)
})
return compressedMeasuredValues
}
// New returns a log file with basic functions for reading and writing data. The
// file extension of the logfile is taken into account to format the logfile
// into the correct format.
@ -124,9 +57,11 @@ func New(logfile string) Logfile {
}
func writeCreationDate(measuredValues []*types.MeasuredValue) error {
now := format.FormatedTime()
for _, measuredValue := range measuredValues {
now := format.FormatedTime()
measuredValue.CreationDate = now
if measuredValue.CreationDate == nil {
measuredValue.CreationDate = &now
}
}
return nil
}

1
pkg/storage/logfile/logfile_test.go
View File

@ -1 +0,0 @@
package logfile_test

70
pkg/storage/storage.go Normal file
View File

@ -0,0 +1,70 @@
package storage
import (
"math"
"sort"
"github.com/go-flucky/flucky/pkg/internal/format"
"github.com/go-flucky/flucky/pkg/types"
)
// Compression the measured values. The system checks whether the measured values
// of the same type correspond to those of the predecessor. If this is the case,
// the current value is discarded and the validity date of the previous value is
// set to that of the current value. This means that no information is lost.
// Only the validity period of the measured value is increased.
func Compression(measuredValues []*types.MeasuredValue) []*types.MeasuredValue {
compressedMeasuredValues := make([]*types.MeasuredValue, 0)
lastMeasuredValuesBySensors := make(map[string]map[types.MeasuredValueType]*types.MeasuredValue, 0)
// Sort all measured values according to the start time of the validity date
// in order to successfully implement the subsequent compression.
sort.SliceStable(measuredValues, func(i int, j int) bool {
return measuredValues[i].FromDate.Before(measuredValues[j].TillDate)
})
now := format.FormatedTime()
for _, measuredValue := range measuredValues {
if _, ok := lastMeasuredValuesBySensors[measuredValue.SensorID]; !ok {
lastMeasuredValuesBySensors[measuredValue.SensorID] = make(map[types.MeasuredValueType]*types.MeasuredValue, 0)
}
if _, ok := lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType]; !ok {
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType] = measuredValue
continue
}
if lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].Value == measuredValue.Value {
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].TillDate = measuredValue.TillDate
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].UpdateDate = &now
} else if lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType].Value != measuredValue.Value {
compressedMeasuredValues = append(compressedMeasuredValues, lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType])
delete(lastMeasuredValuesBySensors[measuredValue.SensorID], measuredValue.ValueType)
lastMeasuredValuesBySensors[measuredValue.SensorID][measuredValue.ValueType] = measuredValue
}
}
// Copy all remaining entries from the map into the cache array
for _, lastMeasuredValuesBySensor := range lastMeasuredValuesBySensors {
for _, measuredValueType := range types.MeasuredValueTypes {
if measuredValue, ok := lastMeasuredValuesBySensor[measuredValueType]; ok {
compressedMeasuredValues = append(compressedMeasuredValues, measuredValue)
}
}
}
// Sort all measured values again to include the measured values from the
// cache.
sort.SliceStable(compressedMeasuredValues, func(i int, j int) bool {
return compressedMeasuredValues[i].FromDate.Before(compressedMeasuredValues[j].FromDate)
})
return compressedMeasuredValues
}
func Round(measuredValues []*types.MeasuredValue, round float64) {
for _, measuredValue := range measuredValues {
measuredValue.Value = math.Round(measuredValue.Value/round) * round
}
}

286
pkg/storage/storage_test.go Normal file
View File

@ -0,0 +1,286 @@
package storage_test
import (
"testing"
"github.com/go-flucky/flucky/pkg/storage"
"github.com/go-flucky/flucky/test/goldenfiles"
"github.com/stretchr/testify/require"
)
type testCase struct {
source string
expected string
compression bool
round float64
}
var (
testCases = []*testCase{
// CSV - Uncompressed Not Rounded -> Uncompressed Rounded 0.5
&testCase{
source: "test/goldenfiles/csv/goldenMeasuredValuesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenMeasuredValuesUncompressedRounded.csv",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenHumiditiesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenHumiditiesUncompressedRounded.csv",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenPressuresUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenPressuresUncompressedRounded.csv",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenTemperaturesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenTemperaturesUncompressedRounded.csv",
compression: false,
round: 0.5,
},
// CSV - Uncompressed Not Rounded -> Compressed Not Rounded
&testCase{
source: "test/goldenfiles/csv/goldenMeasuredValuesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenMeasuredValuesCompressedNotRounded.csv",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/csv/goldenHumiditiesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenHumiditiesCompressedNotRounded.csv",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/csv/goldenPressuresUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenPressuresCompressedNotRounded.csv",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/csv/goldenTemperaturesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenTemperaturesCompressedNotRounded.csv",
compression: true,
round: 0,
},
// CSV - Uncompressed Not Rounded -> Compressed Rounded 0.5
&testCase{
source: "test/goldenfiles/csv/goldenMeasuredValuesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenMeasuredValuesCompressedRounded.csv",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenHumiditiesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenHumiditiesCompressedRounded.csv",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenPressuresUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenPressuresCompressedRounded.csv",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/csv/goldenTemperaturesUncompressedNotRounded.csv",
expected: "test/goldenfiles/csv/goldenTemperaturesCompressedRounded.csv",
compression: true,
round: 0.5,
},
// JSON - Uncompressed Not Rounded -> Uncompressed Rounded 0.5
&testCase{
source: "test/goldenfiles/json/goldenMeasuredValuesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenMeasuredValuesUncompressedRounded.json",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenHumiditiesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenHumiditiesUncompressedRounded.json",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenPressuresUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenPressuresUncompressedRounded.json",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenTemperaturesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenTemperaturesUncompressedRounded.json",
compression: false,
round: 0.5,
},
// JSON - Uncompressed Not Rounded -> Compressed Not Rounded
&testCase{
source: "test/goldenfiles/json/goldenMeasuredValuesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenMeasuredValuesCompressedNotRounded.json",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/json/goldenHumiditiesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenHumiditiesCompressedNotRounded.json",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/json/goldenPressuresUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenPressuresCompressedNotRounded.json",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/json/goldenTemperaturesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenTemperaturesCompressedNotRounded.json",
compression: true,
round: 0,
},
// JSON - Uncompressed Not Rounded -> Compressed Rounded 0.5
&testCase{
source: "test/goldenfiles/json/goldenMeasuredValuesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenMeasuredValuesCompressedRounded.json",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenHumiditiesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenHumiditiesCompressedRounded.json",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenPressuresUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenPressuresCompressedRounded.json",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/json/goldenTemperaturesUncompressedNotRounded.json",
expected: "test/goldenfiles/json/goldenTemperaturesCompressedRounded.json",
compression: true,
round: 0.5,
},
// XML - Uncompressed Not Rounded -> Uncompressed Rounded 0.5
&testCase{
source: "test/goldenfiles/xml/goldenMeasuredValuesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenMeasuredValuesUncompressedRounded.xml",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenHumiditiesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenHumiditiesUncompressedRounded.xml",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenPressuresUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenPressuresUncompressedRounded.xml",
compression: false,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenTemperaturesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenTemperaturesUncompressedRounded.xml",
compression: false,
round: 0.5,
},
// XML - Uncompressed Not Rounded -> Compressed Not Rounded
&testCase{
source: "test/goldenfiles/xml/goldenMeasuredValuesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenMeasuredValuesCompressedNotRounded.xml",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/xml/goldenHumiditiesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenHumiditiesCompressedNotRounded.xml",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/xml/goldenPressuresUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenPressuresCompressedNotRounded.xml",
compression: true,
round: 0,
},
&testCase{
source: "test/goldenfiles/xml/goldenTemperaturesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenTemperaturesCompressedNotRounded.xml",
compression: true,
round: 0,
},
// XML - Uncompressed Not Rounded -> Compressed Rounded 0.5
&testCase{
source: "test/goldenfiles/xml/goldenMeasuredValuesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenMeasuredValuesCompressedRounded.xml",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenHumiditiesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenHumiditiesCompressedRounded.xml",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenPressuresUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenPressuresCompressedRounded.xml",
compression: true,
round: 0.5,
},
&testCase{
source: "test/goldenfiles/xml/goldenTemperaturesUncompressedNotRounded.xml",
expected: "test/goldenfiles/xml/goldenTemperaturesCompressedRounded.xml",
compression: true,
round: 0.5,
},
}
)
func TestCompressionAndRounding(t *testing.T) {
require := require.New(t)
for _, testCase := range testCases {
measuredValues, err := goldenfiles.GetGoldenMeasuredValues(testCase.source)
require.NoError(err)
expectedMeasuredValues, err := goldenfiles.GetGoldenMeasuredValues(testCase.expected)
require.NoError(err)
actualMeasuredValues := measuredValues
if testCase.round != 0 {
storage.Round(actualMeasuredValues, testCase.round)
}
if testCase.compression {
actualMeasuredValues = storage.Compression(actualMeasuredValues)
}
for i, _ := range expectedMeasuredValues {
require.Equal(expectedMeasuredValues[i].ID, actualMeasuredValues[i].ID, "ID of element %v is not equal between expected and actual", i)
require.Equal(expectedMeasuredValues[i].Value, actualMeasuredValues[i].Value, "Value of element %v is not equal between expected and actual", i)
require.Equal(expectedMeasuredValues[i].ValueType, actualMeasuredValues[i].ValueType, "ValueType of element %v is not equal between expected and actual", i)
require.Equal(expectedMeasuredValues[i].FromDate, actualMeasuredValues[i].FromDate, "FromDate of element %v is not equal between expected and actual", i)
require.Equal(expectedMeasuredValues[i].TillDate, actualMeasuredValues[i].TillDate, "TillDate of element %v is not equal between expected and actual", i)
require.Equal(expectedMeasuredValues[i].SensorID, actualMeasuredValues[i].SensorID, "SensorID of element %v is not equal between expected and actual", i)
}
}
}

2
pkg/types/measuredValue.go
View File

@ -12,7 +12,7 @@ type MeasuredValue struct {
FromDate time.Time `json:"from_date" xml:"from_date"`
TillDate time.Time `json:"till_date" xml:"till_date"`
SensorID string `json:"sensor_id" xml:"sensor_id"`
CreationDate time.Time `json:"creation_date" xml:"creation_date"`
CreationDate *time.Time `json:"creation_date" xml:"creation_date"`
UpdateDate *time.Time `json:"update_date" xml:"update_date"`
}