From 4680841684eb6f4bf363200530db3f3276896f7c Mon Sep 17 00:00:00 2001 From: Markus Pesch Date: Thu, 27 Jun 2019 18:44:26 +0200 Subject: [PATCH] fix(pkg/logfile): compression for measured values --- pkg/logfile/json.go | 9 +++++ pkg/logfile/logfile.go | 81 +++++++++++++++++++++++--------------- pkg/types/humidity.go | 24 ++++++++--- pkg/types/measuredValue.go | 15 +++++++ pkg/types/temperature.go | 24 ++++++++--- 5 files changed, 109 insertions(+), 44 deletions(-) diff --git a/pkg/logfile/json.go b/pkg/logfile/json.go index 75ce26e..66a8ef7 100644 --- a/pkg/logfile/json.go +++ b/pkg/logfile/json.go @@ -26,6 +26,10 @@ func (jl *jsonLogfile) Append(compression bool, measuredValues []types.MeasuredV allMeasuredValues = append(allMeasuredValues, measuredValues...) + if compression { + allMeasuredValues = Compression(allMeasuredValues) + } + err = jl.Write(allMeasuredValues) if err != nil { return err @@ -56,6 +60,11 @@ func (jl *jsonLogfile) Read() ([]types.MeasuredValue, error) { measuredValues := make([]types.MeasuredValue, 0) for _, unmarshaledMeasuredValue := range unmarshaledMeasuredValues { + + if unmarshaledMeasuredValue == nil { + continue + } + mappedMeasuredValue, ok := unmarshaledMeasuredValue.(map[string]interface{}) if !ok { log.Println("Can not parse into map") diff --git a/pkg/logfile/logfile.go b/pkg/logfile/logfile.go index c6cc636..cef5d8f 100644 --- a/pkg/logfile/logfile.go +++ b/pkg/logfile/logfile.go @@ -2,6 +2,7 @@ package logfile import ( "path/filepath" + "sort" "time" "github.com/go-flucky/flucky/pkg/types" @@ -10,44 +11,60 @@ import ( // var validUUID = regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$") var timeFormat = time.RFC3339 -// CompressTemperature compresses the temperatures from an array. It is checked -// whether the measured temperature of a value corresponds to that of the -// predecessor. If this is the case, the current value is discarded and the -// validity date of the predecessor value is set to that of the current value. -// No information is lost as a result. The validity period of the measured value -// is thereby exclusively increased. -// func Compression(measuredValues []types) []*types.Temperature { -// compressedTemperatures := make([]*types.Temperature, 0) -// lastTemperatureBySensors := make(map[string]*types.Temperature, 0) +// Compression the measured values. The system checks whether the measured values +// of the same type correspond to those of the predecessor. If this is the case, +// the current value is discarded and the validity date of the previous value is +// set to that of the current value. This means that no information is lost. +// Only the validity period of the measured value is increased. +func Compression(measuredValues []types.MeasuredValue) []types.MeasuredValue { + compressedMeasuredValues := make([]types.MeasuredValue, 0) + lastMeasuredValuesBySensors := make(map[string]map[types.MeasuredValueType]types.MeasuredValue, 0) -// // Sort all measured temperatures beforehand by the starting validity date to -// // avoid errors when compressing the temperatures. -// SortTemperatures(temperatures) + // Sort all measured values according to the start time of the validity date + // in order to successfully implement the subsequent compression. + sort.SliceStable(measuredValues, func(i int, j int) bool { + return measuredValues[i].GetFromDate().Before(measuredValues[j].GetFromDate()) + }) -// for _, temperature := range temperatures { -// if lastTemperatureBySensor, ok := lastTemperatureBySensors[temperature.SensorID]; ok { -// if lastTemperatureBySensor.TemperatureValue == temperature.TemperatureValue { + now := time.Now() -// lastTemperatureBySensors[temperature.SensorID].TemperatureTillDate = temperature.TemperatureTillDate + for _, measuredValue := range measuredValues { + if _, ok := lastMeasuredValuesBySensors[measuredValue.GetSensorID()]; !ok { + lastMeasuredValuesBySensors[measuredValue.GetSensorID()] = make(map[types.MeasuredValueType]types.MeasuredValue, 0) + } -// now := time.Now() -// lastTemperatureBySensors[temperature.SensorID].UpdateDate = &now -// } else { -// compressedTemperatures = append(compressedTemperatures, lastTemperatureBySensors[temperature.SensorID]) -// lastTemperatureBySensors[temperature.SensorID] = temperature -// } -// } else { -// lastTemperatureBySensors[temperature.SensorID] = temperature -// } -// } + if _, ok := lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()]; !ok { + lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()] = measuredValue + continue + } -// // Copy all remaining entries from the map into the array -// for _, lastTemperatureBySensor := range lastTemperatureBySensors { -// compressedTemperatures = append(compressedTemperatures, lastTemperatureBySensor) -// } + if lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].GetValue() == measuredValue.GetValue() { + lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].SetTillDate(measuredValue.GetTillDate()) + lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].SetUpdateDate(&now) + } else if lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].GetValue() != measuredValue.GetValue() { + compressedMeasuredValues = append(compressedMeasuredValues, lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()]) + delete(lastMeasuredValuesBySensors[measuredValue.GetSensorID()], measuredValue.GetMeasuredValueType()) + lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()] = measuredValue + } + } -// return compressedTemperatures -// } + // Copy all remaining entries from the map into the cache array + for _, lastMeasuredValuesBySensor := range lastMeasuredValuesBySensors { + for _, measuredValueType := range types.MeasuredValueTypes { + if measuredValue, ok := lastMeasuredValuesBySensor[measuredValueType]; ok { + compressedMeasuredValues = append(compressedMeasuredValues, measuredValue) + } + } + } + + // Sort all measured values again to include the measured values from the + // cache. + sort.SliceStable(compressedMeasuredValues, func(i int, j int) bool { + return compressedMeasuredValues[i].GetFromDate().Before(compressedMeasuredValues[j].GetFromDate()) + }) + + return compressedMeasuredValues +} // New returns a log file with basic functions for reading and writing data. The // file extension of the logfile is taken into account to format the logfile diff --git a/pkg/types/humidity.go b/pkg/types/humidity.go index 2b930ae..2905a2e 100644 --- a/pkg/types/humidity.go +++ b/pkg/types/humidity.go @@ -13,20 +13,32 @@ type Humidity struct { UpdateDate *time.Time `json:"update_date" xml:"update_date"` } -func (h *Humidity) GetCreationDate() *time.Time { - return h.CreationDate -} - func (h *Humidity) GetID() string { return h.HumidityID } +func (h *Humidity) GetValue() float64 { + return h.HumidityValue +} + +func (h *Humidity) GetFromDate() time.Time { + return h.HumidityFromDate +} + +func (h *Humidity) GetTillDate() time.Time { + return h.HumidityTillDate +} + func (h *Humidity) GetSensorID() string { return h.SensorID } -func (h *Humidity) GetValue() float64 { - return h.HumidityValue +func (h *Humidity) GetCreationDate() *time.Time { + return h.CreationDate +} + +func (h *Humidity) GetMeasuredValueType() MeasuredValueType { + return MeasuredValueTypeHumidity } func (h *Humidity) SetTillDate(date time.Time) { diff --git a/pkg/types/measuredValue.go b/pkg/types/measuredValue.go index df34038..26f0a5e 100644 --- a/pkg/types/measuredValue.go +++ b/pkg/types/measuredValue.go @@ -5,9 +5,24 @@ import "time" type MeasuredValue interface { GetID() string GetCreationDate() *time.Time + GetMeasuredValueType() MeasuredValueType + GetFromDate() time.Time + GetTillDate() time.Time GetSensorID() string GetValue() float64 SetCreationDate(date *time.Time) SetTillDate(date time.Time) SetUpdateDate(date *time.Time) } + +type MeasuredValueType string + +const ( + MeasuredValueTypeHumidity MeasuredValueType = "humidity" + MeasuredValueTypeTemperature MeasuredValueType = "temperature" +) + +var MeasuredValueTypes = []MeasuredValueType{ + MeasuredValueTypeHumidity, + MeasuredValueTypeTemperature, +} diff --git a/pkg/types/temperature.go b/pkg/types/temperature.go index 097db14..2faef2e 100644 --- a/pkg/types/temperature.go +++ b/pkg/types/temperature.go @@ -13,20 +13,32 @@ type Temperature struct { UpdateDate *time.Time `json:"update_date" xml:"update_date"` } -func (t *Temperature) GetCreationDate() *time.Time { - return t.CreationDate -} - func (t *Temperature) GetID() string { return t.TemperatureID } +func (t *Temperature) GetValue() float64 { + return t.TemperatureValue +} + +func (t *Temperature) GetFromDate() time.Time { + return t.TemperatureFromDate +} + +func (t *Temperature) GetTillDate() time.Time { + return t.TemperatureTillDate +} + func (t *Temperature) GetSensorID() string { return t.SensorID } -func (t *Temperature) GetValue() float64 { - return t.TemperatureValue +func (t *Temperature) GetCreationDate() *time.Time { + return t.CreationDate +} + +func (t *Temperature) GetMeasuredValueType() MeasuredValueType { + return MeasuredValueTypeTemperature } func (t *Temperature) SetTillDate(date time.Time) {