fix: use go-migrate pkg to init or update db schema

Instead to implement own logic how the database scheme should be updated
or migrated to a newer or older version flucky use now instead the
go-migrate package.
This commit is contained in:
2021-01-30 15:44:21 +01:00
parent 23695b4513
commit 366dccde12
80 changed files with 843 additions and 167 deletions

View File

@ -7,8 +7,9 @@ import (
"net/url"
"os"
"path/filepath"
"strings"
postgresdml "git.cryptic.systems/volker.raschek/flucky/pkg/repository/db/postgres/dml"
sqlite3dml "git.cryptic.systems/volker.raschek/flucky/pkg/repository/db/sqlite3/dml"
"git.cryptic.systems/volker.raschek/flucky/pkg/types"
"git.cryptic.systems/volker.raschek/go-logger"
)
@ -25,7 +26,7 @@ type Database interface {
InsertOrUpdateMeasuredValues(ctx context.Context, measuredValues ...*types.MeasuredValue) error
InsertSensors(ctx context.Context, sensors ...*types.Sensor) error
InsertOrUpdateSensors(ctx context.Context, sensors ...*types.Sensor) error
Scheme(ctx context.Context) error
Migrate(ctx context.Context) error
SelectDevice(ctx context.Context, deviceID string) (*types.Device, error)
SelectDevices(ctx context.Context) ([]*types.Device, error)
SelectHumidity(ctx context.Context, id string) (*types.MeasuredValue, error)
@ -41,11 +42,11 @@ type Database interface {
}
// New returns a new database backend interface
func New(dsnURL *url.URL, flogger logger.Logger) (Database, error) {
func New(databaseURL *url.URL, flogger logger.Logger) (Database, error) {
// Check of nil pointer
for _, parameter := range []interface{}{
dsnURL,
databaseURL,
flogger,
} {
if parameter == nil {
@ -53,72 +54,78 @@ func New(dsnURL *url.URL, flogger logger.Logger) (Database, error) {
}
}
// Load Queryfiles
queries := make(map[string]string, 0)
for _, asset := range AssetNames() {
if !strings.Contains(asset, dsnURL.Scheme) {
continue
}
body, err := Asset(asset)
if err != nil {
return nil, err
}
queryFile := filepath.Base(asset)
queries[queryFile] = string(body)
}
var (
database Database
err error
)
switch dsnURL.Scheme {
switch databaseURL.Scheme {
case "postgres":
// postgres://[user]:[password]@[host]:[port]/[path]?[query]
newDBO, err := sql.Open(dsnURL.Scheme, dsnURL.String())
newDBO, err := sql.Open(databaseURL.Scheme, databaseURL.String())
if err != nil {
return nil, err
}
queries := make(map[string]string, 0)
for _, assetName := range postgresdml.AssetNames() {
body, err := postgresdml.Asset(assetName)
if err != nil {
return nil, fmt.Errorf("Failed to load asset %v, %w", assetName, err)
}
queries[assetName] = string(body)
}
database = &Postgres{
dbo: newDBO,
flogger: flogger,
queries: queries,
databaseURL: databaseURL,
dbo: newDBO,
flogger: flogger,
queries: queries,
}
case "sqlite3":
// Create directory where the db file will be created if not exists.
if _, err := os.Stat(filepath.Dir(dsnURL.Path)); os.IsNotExist(err) {
err := os.MkdirAll(filepath.Dir(dsnURL.Path), 0755)
// Create directory if not exist
if _, err := os.Stat(filepath.Dir(databaseURL.Path)); os.IsNotExist(err) {
err := os.MkdirAll(filepath.Dir(databaseURL.Path), 0755)
if err != nil {
return nil, err
}
}
// sqlite3:///[path]?[query] flucky dsn
// file:///[path]?[query] sql-lib dsn
newDBO, err := sql.Open(dsnURL.Scheme, fmt.Sprintf("file://%v?%v", dsnURL.Path, dsnURL.RawQuery))
// enable foreign keys
values := databaseURL.Query()
values.Set("_foreign_keys", "on")
customRawURL := fmt.Sprintf("file://%v?%v", databaseURL.Path, values.Encode())
sqlDB, err := sql.Open("sqlite3", customRawURL)
if err != nil {
return nil, err
}
queries := make(map[string]string, 0)
for _, assetName := range sqlite3dml.AssetNames() {
body, err := sqlite3dml.Asset(assetName)
if err != nil {
return nil, fmt.Errorf("Failed to load asset %v, %w", assetName, err)
}
queries[assetName] = string(body)
}
database = &SQLite{
dbo: newDBO,
flogger: flogger,
queries: queries,
databaseURL: databaseURL,
dbo: sqlDB,
flogger: flogger,
queries: queries,
}
default:
return nil, fmt.Errorf("Unsupported database scheme: %v", dsnURL.Scheme)
return nil, fmt.Errorf("Unsupported database scheme: %v", databaseURL.Scheme)
}
// Initialize database scheme if not exists
err = database.Scheme(context.Background())
err = database.Migrate(context.Background())
if err != nil {
return nil, err
}

View File

@ -3,18 +3,24 @@ package db
import (
"context"
"database/sql"
"errors"
"fmt"
"net/url"
"time"
postgresddl "git.cryptic.systems/volker.raschek/flucky/pkg/repository/db/postgres/ddl"
"git.cryptic.systems/volker.raschek/flucky/pkg/types"
"git.cryptic.systems/volker.raschek/go-logger"
"github.com/golang-migrate/migrate/v4"
bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
)
// Postgres implementation
type Postgres struct {
dbo *sql.DB
flogger logger.Logger
queries map[string]string
databaseURL *url.URL
dbo *sql.DB
flogger logger.Logger
queries map[string]string
}
// Close closes the database and prevents new queries from starting. Close then
@ -420,21 +426,29 @@ func (postgres *Postgres) InsertSensors(ctx context.Context, sensors ...*types.S
return tx.Commit()
}
// Scheme creates all required tables if not exist
func (postgres *Postgres) Scheme(ctx context.Context) error {
for _, query := range []string{
postgres.queries["createTableDevices.sql"],
postgres.queries["createTableSensors.sql"],
postgres.queries["createTableHumidities.sql"],
postgres.queries["createTablePressures.sql"],
postgres.queries["createTableTemperatures.sql"],
} {
_, err := postgres.dbo.ExecContext(ctx, query)
if err != nil {
return err
}
// Migrate creates all required tables if not exist
func (postgres *Postgres) Migrate(ctx context.Context) error {
assetSource := bindata.Resource(postgresddl.AssetNames(), func(query string) ([]byte, error) {
return postgresddl.Asset(query)
})
sourceDriver, err := bindata.WithInstance(assetSource)
if err != nil {
return err
}
m, err := migrate.NewWithSourceInstance("bindata", sourceDriver, postgres.databaseURL.String())
if err != nil {
return err
}
err = m.Up()
switch {
case errors.Is(err, migrate.ErrNoChange):
return nil
default:
return err
}
return nil
}
// SelectDevice from database

View File

@ -0,0 +1 @@
DROP TABLE devices;

View File

@ -1,7 +1,8 @@
CREATE TABLE IF NOT EXISTS devices (
device_id CHAR(36) CONSTRAINT pk_devices PRIMARY KEY,
CREATE TABLE devices (
device_id CHAR(36) NOT NULL,
device_name VARCHAR(64) NOT NULL,
device_location VARCHAR(64),
creation_date TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP WITH TIME ZONE
update_date TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_devices PRIMARY KEY(device_id)
);

View File

@ -0,0 +1 @@
DROP TABLE sensors;

View File

@ -1,5 +1,5 @@
CREATE TABLE IF NOT EXISTS sensors (
sensor_id CHAR(36) CONSTRAINT pk_sensors PRIMARY KEY,
CREATE TABLE sensors (
sensor_id CHAR(36) NOT NULL,
sensor_name VARCHAR(64) NOT NULL,
sensor_location VARCHAR(64),
wire_id VARCHAR(64),
@ -11,11 +11,7 @@ CREATE TABLE IF NOT EXISTS sensors (
tick_duration VARCHAR(6) NOT NULL,
device_id CHAR(36) NOT NULL,
creation_date TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP WITH TIME ZONE
);
ALTER TABLE sensors
ADD FOREIGN KEY (device_id)
REFERENCES devices(device_id)
ON DELETE CASCADE
ON UPDATE CASCADE;
update_date TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_sensors PRIMARY KEY(sensor_id),
CONSTRAINT fk_sensors_device_id FOREIGN KEY (device_id) REFERENCES devices(device_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE humidities;

View File

@ -1,14 +1,10 @@
CREATE TABLE IF NOT EXISTS pressures (
id CHAR(36) CONSTRAINT pk_pressures PRIMARY KEY,
CREATE TABLE humidities (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP WITH TIME ZONE NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP WITH TIME ZONE
);
ALTER TABLE pressures
ADD FOREIGN KEY (sensor_id)
REFERENCES sensors(sensor_id)
ON DELETE CASCADE
ON UPDATE CASCADE;
update_date TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_humidites PRIMARY KEY (id),
CONSTRAINT pk_humidites_sensor_id FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE pressures;

View File

@ -1,14 +1,10 @@
CREATE TABLE IF NOT EXISTS temperatures (
id CHAR(36) CONSTRAINT pk_temperatures PRIMARY KEY,
CREATE TABLE pressures (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP WITH TIME ZONE NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP WITH TIME ZONE
);
ALTER TABLE temperatures
ADD FOREIGN KEY (sensor_id)
REFERENCES sensors(sensor_id)
ON DELETE CASCADE
ON UPDATE CASCADE;
update_date TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_pressures PRIMARY KEY(id),
CONSTRAINT pk_pressures_sensor_id FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE temperatures;

View File

@ -1,14 +1,10 @@
CREATE TABLE IF NOT EXISTS humidities (
id CHAR(36) CONSTRAINT pk_humidities PRIMARY KEY,
CREATE TABLE temperatures (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP WITH TIME ZONE NOT NULL,
date TIMESTAMP WITH TIME ZONE NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP WITH TIME ZONE
);
ALTER TABLE humidities
ADD FOREIGN KEY (sensor_id)
REFERENCES sensors(sensor_id)
ON DELETE CASCADE
ON UPDATE CASCADE;
update_date TIMESTAMP WITH TIME ZONE,
CONSTRAINT pk_temperatures PRIMARY KEY (id),
CONSTRAINT fk_temperatures_sensor_id FOREIGN KEY (sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -3,17 +3,23 @@ package db
import (
"context"
"database/sql"
"errors"
"fmt"
"net/url"
sqlite3ddl "git.cryptic.systems/volker.raschek/flucky/pkg/repository/db/sqlite3/ddl"
"git.cryptic.systems/volker.raschek/flucky/pkg/types"
"git.cryptic.systems/volker.raschek/go-logger"
"github.com/golang-migrate/migrate/v4"
bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
)
// SQLite implementation
type SQLite struct {
dbo *sql.DB
flogger logger.Logger
queries map[string]string
databaseURL *url.URL
dbo *sql.DB
flogger logger.Logger
queries map[string]string
}
// Close closes the database and prevents new queries from starting. Close then
@ -410,21 +416,29 @@ func (sqlite *SQLite) InsertSensors(ctx context.Context, sensors ...*types.Senso
return tx.Commit()
}
// Scheme creates all required tables if not exist
func (sqlite *SQLite) Scheme(ctx context.Context) error {
for _, query := range []string{
sqlite.queries["createTableDevices.sql"],
sqlite.queries["createTableSensors.sql"],
sqlite.queries["createTableHumidities.sql"],
sqlite.queries["createTablePressures.sql"],
sqlite.queries["createTableTemperatures.sql"],
} {
_, err := sqlite.dbo.ExecContext(ctx, query)
if err != nil {
return err
}
// Migrate creates all required tables if not exist
func (sqlite *SQLite) Migrate(ctx context.Context) error {
assetSource := bindata.Resource(sqlite3ddl.AssetNames(), func(query string) ([]byte, error) {
return sqlite3ddl.Asset(query)
})
sourceDriver, err := bindata.WithInstance(assetSource)
if err != nil {
return err
}
m, err := migrate.NewWithSourceInstance("bindata", sourceDriver, sqlite.databaseURL.String())
if err != nil {
return err
}
err = m.Up()
switch {
case errors.Is(err, migrate.ErrNoChange):
return nil
default:
return err
}
return nil
}
// SelectDevice from database

View File

@ -1,7 +0,0 @@
CREATE TABLE IF NOT EXISTS devices (
device_id CHAR(36) PRIMARY KEY,
device_name VARCHAR(64) NOT NULL,
device_location VARCHAR(64),
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP
);

View File

@ -1,9 +0,0 @@
CREATE TABLE IF NOT EXISTS humidities (
id CHAR(36) PRIMARY KEY,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP,
FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id)
);

View File

@ -1,9 +0,0 @@
CREATE TABLE IF NOT EXISTS pressures (
id CHAR(36) PRIMARY KEY,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP,
FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id)
);

View File

@ -1,9 +0,0 @@
CREATE TABLE IF NOT EXISTS temperatures (
id CHAR(36) PRIMARY KEY,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
update_date TIMESTAMP,
FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id)
);

View File

@ -0,0 +1 @@
DROP TABLE devices;

View File

@ -0,0 +1,8 @@
CREATE TABLE devices (
device_id CHAR(36) NOT NULL,
device_name VARCHAR(64) NOT NULL,
device_location VARCHAR(64),
creation_date TIMESTAMP DEFAULT (datetime('now', 'localtime')) NOT NULL,
update_date TIMESTAMP,
CONSTRAINT pk_devices PRIMARY KEY (device_id)
);

View File

@ -0,0 +1 @@
DROP TABLE sensors;

View File

@ -1,16 +1,17 @@
CREATE TABLE IF NOT EXISTS sensors (
sensor_id CHAR(36) PRIMARY KEY,
sensor_name VARCHAR(64) NOT NULL,
CREATE TABLE sensors (
sensor_id CHAR(36) NOT NULL,
sensor_name VARCHAR(64) NOT NULL,
sensor_location VARCHAR(64),
wire_id VARCHAR(64),
i2c_bus VARCHAR(255),
i2c_address VARCHAR(12),
gpio_number VARCHAR(6),
sensor_model VARCHAR(16) NOT NULL,
sensor_enabled INTEGER(1) DEFAULT 1 NOT NULL,
tick_duration VARCHAR(6) NOT NULL,
device_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
sensor_model VARCHAR(16) NOT NULL,
sensor_enabled INTEGER(1) DEFAULT 1 NOT NULL,
tick_duration VARCHAR(6) NOT NULL,
device_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT (datetime('now', 'localtime')) NOT NULL,
update_date TIMESTAMP,
FOREIGN KEY(device_id) REFERENCES devices(device_id)
CONSTRAINT pk_sensors PRIMARY KEY(sensor_id),
CONSTRAINT fk_sensors_device_id FOREIGN KEY(device_id) REFERENCES devices(device_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE humidities;

View File

@ -0,0 +1,10 @@
CREATE TABLE humidities (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT (datetime('now', 'localtime')) NOT NULL,
update_date TIMESTAMP,
CONSTRAINT pk_humidities PRIMARY KEY(id),
CONSTRAINT fk_humidities_sensor_id FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE humidities;

View File

@ -0,0 +1,10 @@
CREATE TABLE pressures (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT (datetime('now', 'localtime')) NOT NULL,
update_date TIMESTAMP,
CONSTRAINT pk_pressures PRIMARY KEY(id),
CONSTRAINT fk_pressures_sensor_id FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -0,0 +1 @@
DROP TABLE temperatures;

View File

@ -0,0 +1,10 @@
CREATE TABLE temperatures (
id CHAR(36) NOT NULL,
value NUMERIC(10,3) NOT NULL,
date TIMESTAMP NOT NULL,
sensor_id CHAR(36) NOT NULL,
creation_date TIMESTAMP DEFAULT (datetime('now', 'localtime')) NOT NULL,
update_date TIMESTAMP,
CONSTRAINT pk_temperatures PRIMARY KEY(id),
CONSTRAINT fk_temperatures_id FOREIGN KEY(sensor_id) REFERENCES sensors(sensor_id) ON DELETE CASCADE ON UPDATE CASCADE
);

View File

@ -19,6 +19,8 @@ import (
uuid "github.com/satori/go.uuid"
"github.com/stretchr/testify/require"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/database/sqlite3"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
)