feat: add pkg to start postgres container

This commit is contained in:
Markus Pesch 2020-06-14 22:08:42 +02:00
parent fa45125b49
commit ec7cad4555
Signed by: volker.raschek
GPG Key ID: 852BCC170D81A982
3 changed files with 618 additions and 1 deletions

View File

@ -0,0 +1,285 @@
package database
import (
"context"
"database/sql"
"errors"
"fmt"
"net/url"
"strconv"
"strings"
"testing"
"time"
"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/dockerutils"
"github.com/docker/docker/api/types"
uuid "github.com/satori/go.uuid"
"github.com/stretchr/testify/require"
)
var (
ErrInvalidAttr = errors.New("Invalid DatabaseOption attribute")
)
type PostgresOptions struct {
ContainerEnv map[string]string
ContainerImage string
ContainerLabels map[string]string
ContainerName string
ContainerNetworks map[string][]string
ContainerNetworkLabels map[string]map[string]string
ContainerPort string
Driver string
DSN string
HostPort string
}
// Validate the DatabaseOption struct, if all required atrributes are valid
func (dbOptions *PostgresOptions) Validate() error {
// Required strings
for _, values := range [][]string{
0: {"ContainerImage", dbOptions.ContainerImage},
1: {"ContainerPort", dbOptions.ContainerPort},
2: {"Driver", dbOptions.Driver},
3: {"HostPort", dbOptions.HostPort},
} {
if len(values[1]) <= 0 {
return fmt.Errorf("%w: Attribute %v is empty", ErrInvalidAttr, values[0])
}
}
// Require initialized maps
for key, value := range map[string]interface{}{
"ContainerEnv": dbOptions.ContainerEnv,
"ContainerLabels": dbOptions.ContainerLabels,
"ContainerNetworks": dbOptions.ContainerNetworks,
"ContainerNetworkLabels": dbOptions.ContainerNetworkLabels,
} {
if value == nil {
return fmt.Errorf("%w: Attribut %v is not initialized", ErrInvalidAttr, key)
}
}
// Required postgres environment variables
for _, key := range []string{
"POSTGRES_PASSWORD",
"POSTGRES_USER",
"POSTGRES_DB",
} {
if _, present := dbOptions.ContainerEnv[key]; !present {
return fmt.Errorf("%w: Required env %v not defined", ErrInvalidAttr, key)
}
}
// Supported drivers
found := false
for _, supportedDriver := range []string{
"postgres",
} {
if supportedDriver == dbOptions.Driver {
found = true
break
}
}
if !found {
return fmt.Errorf("%w: Driver %v not supported", ErrInvalidAttr, dbOptions.Driver)
}
// Protect well-known ports
hostPort, err := strconv.ParseInt(dbOptions.HostPort, 10, 64)
if err != nil {
return fmt.Errorf("Failed to parse hostport %v: %w", dbOptions.HostPort, err)
}
if hostPort > 0 && hostPort < 1024 {
return fmt.Errorf("%w: ContainerPort %v: Protect well-known ports between 1-1023", ErrInvalidAttr, dbOptions.HostPort)
}
return nil
}
// NewPostgresDatabase starts a new postgres database based on default values
func NewPostgresDatabase(t *testing.T) (*PostgresOptions, func()) {
return StartPostgres(t, NewPostgresOptions())
}
// NewPostgresOptions returns a new PostgresOptions structs with default values
func NewPostgresOptions() *PostgresOptions {
return &PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "postgres",
"POSTGRES_USER": "postgres",
"POSTGRES_DB": "postgres",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerLabels: make(map[string]string, 0),
ContainerName: uuid.NewV4().String(),
ContainerNetworks: make(map[string][]string, 0),
ContainerNetworkLabels: make(map[string]map[string]string, 0),
ContainerPort: "5432",
Driver: "postgres",
DSN: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
HostPort: "0",
}
}
// StartPostgres starts a postgres container image based on the PostgresOption
// struct
func StartPostgres(t *testing.T, pgOptions *PostgresOptions) (*PostgresOptions, func()) {
var (
ctx = context.Background()
require = require.New(t)
cleanupLabels = map[string]string{
uuid.NewV4().String(): uuid.NewV4().String(),
}
)
dockerClient, err := dockerutils.New()
require.NoError(err)
// Validate the PostgresOptions struct
require.NoError(pgOptions.Validate())
// Pre-Define DSN based on the pgOption attributes
pgOptions.DSN = fmt.Sprintf("postgres://%v:%v@localhost:%v/%v?sslmode=disable",
pgOptions.ContainerEnv["POSTGRES_USER"],
pgOptions.ContainerEnv["POSTGRES_PASSWORD"],
pgOptions.HostPort,
pgOptions.ContainerEnv["POSTGRES_DB"],
)
// Create missing networks
// The dockerutils package check only if the network exist and append the
// container with his aliasses as network endpoint. Additionally every network
// get his own network labels. If no network labels are defined, new labels
// will be generated. This is required to remove the networks by their labels.
networks, err := dockerClient.NetworkList(ctx, types.NetworkListOptions{})
require.NoError(err)
for networkName := range pgOptions.ContainerNetworks {
found := false
for _, network := range networks {
if network.Name == networkName {
found = true
break
}
}
if !found {
// Create network labels if there are no one defined
if _, present := pgOptions.ContainerNetworkLabels[networkName]; !present {
pgOptions.ContainerNetworkLabels[networkName] = make(map[string]string, 0)
}
// Append cleanup labels to the network
for key, value := range cleanupLabels {
pgOptions.ContainerNetworkLabels[networkName][key] = value
}
_, err := dockerClient.NetworkCreate(ctx, networkName, types.NetworkCreate{
Labels: pgOptions.ContainerNetworkLabels[networkName],
})
require.NoError(err)
}
}
// Append cleanup labels to the container
for key, value := range cleanupLabels {
pgOptions.ContainerLabels[key] = value
}
// Build database container
databaseContainerBuilder := dockerClient.NewBuilder(pgOptions.ContainerImage).
Env(pgOptions.ContainerEnv).
Labels(pgOptions.ContainerLabels).
Port(fmt.Sprintf("%s:%s", pgOptions.HostPort, pgOptions.ContainerPort)).
Pull().
WithName(pgOptions.ContainerName)
networkNames := make([]string, 0)
for networkName, aliasses := range pgOptions.ContainerNetworks {
if aliasses == nil {
aliasses = make([]string, 0)
}
if len(aliasses) <= 0 {
aliasses = append(aliasses, pgOptions.ContainerName)
}
databaseContainerBuilder.Network(networkName, aliasses...)
pgOptions.ContainerNetworks[networkName] = aliasses
networkNames = append(networkNames, networkName)
}
dbContainerID, err := databaseContainerBuilder.Start(ctx)
require.NoError(err)
// cleanupFunction to remove the database container with all defined networks.
// Skip network if the network has an additional endpoint
cleanupFunc := func() {
err := dockerClient.ContainerRemove(ctx, dbContainerID, types.ContainerRemoveOptions{Force: true})
require.NoError(err)
for networkName := range pgOptions.ContainerNetworks {
networks, err := dockerClient.NetworkListByLabels(ctx, pgOptions.ContainerNetworkLabels[networkName])
require.NoError(err)
for _, network := range networks {
if len(network.Containers) <= 0 {
err := dockerClient.NetworkRemove(ctx, network.ID)
require.NoError(err)
}
}
}
}
// Search for allocated port if public port is defined as random by 0
if pgOptions.HostPort == "0" {
containers, err := dockerClient.ContainerListByLabels(ctx, true, pgOptions.ContainerLabels)
require.NoError(err)
var pubPort string
for _, port := range containers[0].Ports {
if port.PrivatePort == 5432 {
pubPort = strconv.Itoa(int(port.PublicPort))
break
}
}
require.Greater(len(pubPort), 0, "Failed to detect allocated random port of the postgres container")
connectionURL, err := url.Parse(pgOptions.DSN)
require.NoError(err)
parts := strings.Split(connectionURL.Host, ":")
connectionURL.Host = fmt.Sprintf("%v:%v", parts[0], pubPort)
pgOptions.DSN = connectionURL.String()
pgOptions.HostPort = pubPort
}
var db *sql.DB
for i := 0; i < 10; i++ {
db, err = sql.Open(pgOptions.Driver, pgOptions.DSN)
if err != nil {
t.Log("Database not ready - wait 10 seconds")
time.Sleep(10 * time.Second)
continue
}
err = db.Ping()
if err != nil {
t.Log("Database not ready - wait 10 seconds")
db.Close()
time.Sleep(10 * time.Second)
continue
}
break
}
require.NoError(err)
err = db.Close()
require.NoError(err)
return pgOptions, cleanupFunc
}

View File

@ -0,0 +1,331 @@
package database_test
import (
"context"
"database/sql"
"errors"
"fmt"
"testing"
"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/database"
"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/dockerutils"
uuid "github.com/satori/go.uuid"
"github.com/stretchr/testify/require"
_ "github.com/lib/pq"
)
func requireCountNetworks(ctx context.Context, dockerClient *dockerutils.Client, dbOptions *database.PostgresOptions, equal int, t *testing.T) {
require := require.New(t)
containers, err := dockerClient.ContainerListByLabels(ctx, true, dbOptions.ContainerLabels)
require.NoError(err)
require.Equal(equal, len(containers[0].NetworkSettings.Networks))
}
func requireContainerHasNetworksAndAliasses(ctx context.Context, dockerClient *dockerutils.Client, dbOptions *database.PostgresOptions, t *testing.T) {
require := require.New(t)
containers, err := dockerClient.ContainerListByNames(ctx, true, dbOptions.ContainerName)
require.NoError(err)
require.Contains(containers[0].Names, fmt.Sprintf("/%v", dbOptions.ContainerName))
for networkName, _ := range dbOptions.ContainerNetworks {
networks, err := dockerClient.NetworkListByNames(ctx, networkName)
require.NoError(err)
require.Equal(networkName, networks[0].Name)
require.Equal(dbOptions.ContainerName, networks[0].Containers[containers[0].ID].Name)
}
}
func requireTestCleanup(ctx context.Context, dockerClient *dockerutils.Client, cleanupLabels map[string]string, t *testing.T) {
require := require.New(t)
containers, err := dockerClient.ContainerListByLabels(ctx, true, cleanupLabels)
require.NoError(err)
require.NotNil(containers)
require.Len(containers, 0)
networks, err := dockerClient.NetworkListByLabels(ctx, cleanupLabels)
require.NoError(err)
require.NotNil(networks)
require.Len(networks, 0)
}
func TestPostgresDatabase(t *testing.T) {
require := require.New(t)
dbOptions, cleanup := database.NewPostgresDatabase(t)
t.Cleanup(func() { cleanup() })
require.NotEqual(0, dbOptions.HostPort)
dockerClient, err := dockerutils.New()
require.NoError(err)
dbContainers, err := dockerClient.ContainerListByLabels(context.Background(), true, dbOptions.ContainerLabels)
require.NoError(err)
require.NotNil(dbContainers)
require.Len(dbContainers, 1)
dbo, err := sql.Open(dbOptions.Driver, dbOptions.DSN)
require.NoError(err)
require.NoError(dbo.Ping())
query := "SELECT 1"
row := dbo.QueryRow(query)
var i int
err = row.Scan(&i)
require.NoError(err)
require.Equal(1, i)
}
func TestPostgresNetwork(t *testing.T) {
var (
ctx = context.Background()
require = require.New(t)
cleanupLabels = map[string]string{
uuid.NewV4().String(): uuid.NewV4().String(),
}
)
dockerClient, err := dockerutils.New()
require.NoError(err)
t.Cleanup(func() {
dockerClient.ContainerRemoveByLabels(ctx, cleanupLabels)
dockerClient.NetworkRemoveByLabels(ctx, cleanupLabels)
dockerClient.Close()
})
// TestDefaultNetwork
dbOptionsPointer, cleanup := database.NewPostgresDatabase(t)
t.Cleanup(cleanup)
require.NotNil(dbOptionsPointer.ContainerNetworks)
require.Len(dbOptionsPointer.ContainerNetworks, 0)
requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptionsPointer, t)
// TestRandomNetwork
dbOptions := database.NewPostgresOptions()
dbOptions.HostPort = "0"
dbOptions.ContainerName = "Volker"
dbOptions.ContainerLabels = cleanupLabels
dbOptions.ContainerNetworks[uuid.NewV4().String()] = []string{
uuid.NewV4().String(),
uuid.NewV4().String(),
}
_, cleanup = database.StartPostgres(t, dbOptions)
t.Cleanup(cleanup)
requireCountNetworks(ctx, dockerClient, dbOptions, 1, t)
requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptions, t)
// TestRandomMultipleNetworks
containerNetworkNameA := uuid.NewV4().String()
containerNetworkNameB := uuid.NewV4().String()
containerNetworks := map[string][]string{
containerNetworkNameA: {
uuid.NewV4().String(),
uuid.NewV4().String(),
},
containerNetworkNameB: {
uuid.NewV4().String(),
uuid.NewV4().String(),
},
}
containerNetworkLabels := map[string]map[string]string{
containerNetworkNameA: cleanupLabels,
containerNetworkNameB: cleanupLabels,
}
dbOptions = database.NewPostgresOptions()
dbOptions.HostPort = "0"
dbOptions.ContainerName = "Raschek"
dbOptions.ContainerNetworks = containerNetworks
dbOptions.ContainerNetworkLabels = containerNetworkLabels
dbOptions.ContainerLabels = cleanupLabels
_, cleanup = database.StartPostgres(t, dbOptions)
t.Cleanup(cleanup)
requireCountNetworks(ctx, dockerClient, dbOptions, 2, t)
requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptions, t)
// TestCleanup
// Test if a network with an additional endpoint will not be removed
containerNetwork := uuid.NewV4().String()
containerNetworkA := map[string][]string{
containerNetwork: {
uuid.NewV4().String(),
uuid.NewV4().String(),
},
}
containerNetworkB := map[string][]string{
containerNetwork: {
uuid.NewV4().String(),
uuid.NewV4().String(),
},
}
containerNetworkLabels = map[string]map[string]string{
containerNetwork: cleanupLabels,
}
dbOptions = database.NewPostgresOptions()
dbOptions.HostPort = "0"
dbOptions.ContainerName = "is"
dbOptions.ContainerLabels = cleanupLabels
dbOptions.ContainerNetworks = containerNetworkA
dbOptions.ContainerNetworkLabels = containerNetworkLabels
_, cleanup = database.StartPostgres(t, dbOptions)
dbOptions = database.NewPostgresOptions()
dbOptions.HostPort = "0"
dbOptions.ContainerName = "the_best"
dbOptions.ContainerLabels = cleanupLabels
dbOptions.ContainerNetworks = containerNetworkB
dbOptions.ContainerNetworkLabels = containerNetworkLabels
_, cleanupSecond := database.StartPostgres(t, dbOptions)
cleanup()
networkResources, err := dockerClient.NetworkListByNames(ctx, containerNetwork)
require.NoError(err)
require.Len(networkResources[0].Containers, 1)
cleanupSecond()
requireTestCleanup(ctx, dockerClient, cleanupLabels, t)
}
func TestPostgresOptionsValidate(t *testing.T) {
require := require.New(t)
// ContainerImage is empty
dbOptions := database.PostgresOptions{}
err := dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Attribute ContainerImage is empty")
// ContainerPort is empty
dbOptions = database.PostgresOptions{
ContainerImage: "docker.io/library/postgres:13-alpine",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Attribute ContainerPort is empty")
// Driver is empty
dbOptions = database.PostgresOptions{
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Attribute Driver is empty")
// HostPort is empty
dbOptions = database.PostgresOptions{
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "sdfsdfsdf",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Attribute HostPort is empty")
// ContainerEnv: POSTGRES_PASSWORD
dbOptions = database.PostgresOptions{
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "sdfsdfsdf",
HostPort: "asdasd",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_PASSWORD not defined")
// ContainerEnv: POSTGRES_USER
dbOptions = database.PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "HelloWorld",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "sdfsdfsdf",
HostPort: "asdasd",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_USER not defined")
// ContainerEnv: POSTGRES_USER
dbOptions = database.PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "postgres",
"POSTGRES_USER": "postgres",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "sdfsdfsdf",
HostPort: "asdasd",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_DB not defined")
// Driver not supported
dbOptions = database.PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "postgres",
"POSTGRES_USER": "postgres",
"POSTGRES_DB": "postgres",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "sdfsdfsdf",
HostPort: "asdasd",
}
err = dbOptions.Validate()
require.Error(err)
require.True(errors.Is(err, database.ErrInvalidAttr))
require.EqualError(err, "Invalid DatabaseOption attribute: Driver sdfsdfsdf not supported")
// HostPort: Failed to parse
dbOptions = database.PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "postgres",
"POSTGRES_USER": "postgres",
"POSTGRES_DB": "postgres",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "postgres",
HostPort: "asdasd",
}
err = dbOptions.Validate()
require.Error(err)
require.EqualError(err, "Failed to parse hostport asdasd: strconv.ParseInt: parsing \"asdasd\": invalid syntax")
// HostPort: Well-Known port
dbOptions = database.PostgresOptions{
ContainerEnv: map[string]string{
"POSTGRES_PASSWORD": "postgres",
"POSTGRES_USER": "postgres",
"POSTGRES_DB": "postgres",
},
ContainerImage: "docker.io/library/postgres:13-alpine",
ContainerPort: "5432",
Driver: "postgres",
HostPort: "443",
}
err = dbOptions.Validate()
require.Error(err)
require.EqualError(err, "Invalid DatabaseOption attribute: ContainerPort 443: Protect well-known ports between 1-1023")
}

View File

@ -31,7 +31,8 @@ func (client *Client) Close() error {
if client.watcher != nil {
client.watcher.stop()
}
return client.Close()
return client.Client.Close()
}
// ContainerListByLabels returns only containers which match by given labels