fix(pkg/logfile): compression for measured values

This commit is contained in:
Markus Pesch 2019-06-27 18:44:26 +02:00
parent 4d7408ac81
commit 4680841684
Signed by: volker.raschek
GPG Key ID: 852BCC170D81A982
5 changed files with 109 additions and 44 deletions

View File

@ -26,6 +26,10 @@ func (jl *jsonLogfile) Append(compression bool, measuredValues []types.MeasuredV
allMeasuredValues = append(allMeasuredValues, measuredValues...)
if compression {
allMeasuredValues = Compression(allMeasuredValues)
}
err = jl.Write(allMeasuredValues)
if err != nil {
return err
@ -56,6 +60,11 @@ func (jl *jsonLogfile) Read() ([]types.MeasuredValue, error) {
measuredValues := make([]types.MeasuredValue, 0)
for _, unmarshaledMeasuredValue := range unmarshaledMeasuredValues {
if unmarshaledMeasuredValue == nil {
continue
}
mappedMeasuredValue, ok := unmarshaledMeasuredValue.(map[string]interface{})
if !ok {
log.Println("Can not parse into map")

View File

@ -2,6 +2,7 @@ package logfile
import (
"path/filepath"
"sort"
"time"
"github.com/go-flucky/flucky/pkg/types"
@ -10,44 +11,60 @@ import (
// var validUUID = regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
var timeFormat = time.RFC3339
// CompressTemperature compresses the temperatures from an array. It is checked
// whether the measured temperature of a value corresponds to that of the
// predecessor. If this is the case, the current value is discarded and the
// validity date of the predecessor value is set to that of the current value.
// No information is lost as a result. The validity period of the measured value
// is thereby exclusively increased.
// func Compression(measuredValues []types) []*types.Temperature {
// compressedTemperatures := make([]*types.Temperature, 0)
// lastTemperatureBySensors := make(map[string]*types.Temperature, 0)
// Compression the measured values. The system checks whether the measured values
// of the same type correspond to those of the predecessor. If this is the case,
// the current value is discarded and the validity date of the previous value is
// set to that of the current value. This means that no information is lost.
// Only the validity period of the measured value is increased.
func Compression(measuredValues []types.MeasuredValue) []types.MeasuredValue {
compressedMeasuredValues := make([]types.MeasuredValue, 0)
lastMeasuredValuesBySensors := make(map[string]map[types.MeasuredValueType]types.MeasuredValue, 0)
// // Sort all measured temperatures beforehand by the starting validity date to
// // avoid errors when compressing the temperatures.
// SortTemperatures(temperatures)
// Sort all measured values according to the start time of the validity date
// in order to successfully implement the subsequent compression.
sort.SliceStable(measuredValues, func(i int, j int) bool {
return measuredValues[i].GetFromDate().Before(measuredValues[j].GetFromDate())
})
// for _, temperature := range temperatures {
// if lastTemperatureBySensor, ok := lastTemperatureBySensors[temperature.SensorID]; ok {
// if lastTemperatureBySensor.TemperatureValue == temperature.TemperatureValue {
now := time.Now()
// lastTemperatureBySensors[temperature.SensorID].TemperatureTillDate = temperature.TemperatureTillDate
for _, measuredValue := range measuredValues {
if _, ok := lastMeasuredValuesBySensors[measuredValue.GetSensorID()]; !ok {
lastMeasuredValuesBySensors[measuredValue.GetSensorID()] = make(map[types.MeasuredValueType]types.MeasuredValue, 0)
}
// now := time.Now()
// lastTemperatureBySensors[temperature.SensorID].UpdateDate = &now
// } else {
// compressedTemperatures = append(compressedTemperatures, lastTemperatureBySensors[temperature.SensorID])
// lastTemperatureBySensors[temperature.SensorID] = temperature
// }
// } else {
// lastTemperatureBySensors[temperature.SensorID] = temperature
// }
// }
if _, ok := lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()]; !ok {
lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()] = measuredValue
continue
}
// // Copy all remaining entries from the map into the array
// for _, lastTemperatureBySensor := range lastTemperatureBySensors {
// compressedTemperatures = append(compressedTemperatures, lastTemperatureBySensor)
// }
if lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].GetValue() == measuredValue.GetValue() {
lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].SetTillDate(measuredValue.GetTillDate())
lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].SetUpdateDate(&now)
} else if lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()].GetValue() != measuredValue.GetValue() {
compressedMeasuredValues = append(compressedMeasuredValues, lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()])
delete(lastMeasuredValuesBySensors[measuredValue.GetSensorID()], measuredValue.GetMeasuredValueType())
lastMeasuredValuesBySensors[measuredValue.GetSensorID()][measuredValue.GetMeasuredValueType()] = measuredValue
}
}
// return compressedTemperatures
// }
// Copy all remaining entries from the map into the cache array
for _, lastMeasuredValuesBySensor := range lastMeasuredValuesBySensors {
for _, measuredValueType := range types.MeasuredValueTypes {
if measuredValue, ok := lastMeasuredValuesBySensor[measuredValueType]; ok {
compressedMeasuredValues = append(compressedMeasuredValues, measuredValue)
}
}
}
// Sort all measured values again to include the measured values from the
// cache.
sort.SliceStable(compressedMeasuredValues, func(i int, j int) bool {
return compressedMeasuredValues[i].GetFromDate().Before(compressedMeasuredValues[j].GetFromDate())
})
return compressedMeasuredValues
}
// New returns a log file with basic functions for reading and writing data. The
// file extension of the logfile is taken into account to format the logfile

View File

@ -13,20 +13,32 @@ type Humidity struct {
UpdateDate *time.Time `json:"update_date" xml:"update_date"`
}
func (h *Humidity) GetCreationDate() *time.Time {
return h.CreationDate
}
func (h *Humidity) GetID() string {
return h.HumidityID
}
func (h *Humidity) GetValue() float64 {
return h.HumidityValue
}
func (h *Humidity) GetFromDate() time.Time {
return h.HumidityFromDate
}
func (h *Humidity) GetTillDate() time.Time {
return h.HumidityTillDate
}
func (h *Humidity) GetSensorID() string {
return h.SensorID
}
func (h *Humidity) GetValue() float64 {
return h.HumidityValue
func (h *Humidity) GetCreationDate() *time.Time {
return h.CreationDate
}
func (h *Humidity) GetMeasuredValueType() MeasuredValueType {
return MeasuredValueTypeHumidity
}
func (h *Humidity) SetTillDate(date time.Time) {

View File

@ -5,9 +5,24 @@ import "time"
type MeasuredValue interface {
GetID() string
GetCreationDate() *time.Time
GetMeasuredValueType() MeasuredValueType
GetFromDate() time.Time
GetTillDate() time.Time
GetSensorID() string
GetValue() float64
SetCreationDate(date *time.Time)
SetTillDate(date time.Time)
SetUpdateDate(date *time.Time)
}
type MeasuredValueType string
const (
MeasuredValueTypeHumidity MeasuredValueType = "humidity"
MeasuredValueTypeTemperature MeasuredValueType = "temperature"
)
var MeasuredValueTypes = []MeasuredValueType{
MeasuredValueTypeHumidity,
MeasuredValueTypeTemperature,
}

View File

@ -13,20 +13,32 @@ type Temperature struct {
UpdateDate *time.Time `json:"update_date" xml:"update_date"`
}
func (t *Temperature) GetCreationDate() *time.Time {
return t.CreationDate
}
func (t *Temperature) GetID() string {
return t.TemperatureID
}
func (t *Temperature) GetValue() float64 {
return t.TemperatureValue
}
func (t *Temperature) GetFromDate() time.Time {
return t.TemperatureFromDate
}
func (t *Temperature) GetTillDate() time.Time {
return t.TemperatureTillDate
}
func (t *Temperature) GetSensorID() string {
return t.SensorID
}
func (t *Temperature) GetValue() float64 {
return t.TemperatureValue
func (t *Temperature) GetCreationDate() *time.Time {
return t.CreationDate
}
func (t *Temperature) GetMeasuredValueType() MeasuredValueType {
return MeasuredValueTypeTemperature
}
func (t *Temperature) SetTillDate(date time.Time) {