feat: add pkg to start postgres container

pkg/testutils/database/database.go

@@ -0,0 +1,285 @@
+package database
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/dockerutils"
+	"github.com/docker/docker/api/types"
+	uuid "github.com/satori/go.uuid"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	ErrInvalidAttr = errors.New("Invalid DatabaseOption attribute")
+)
+
+type PostgresOptions struct {
+	ContainerEnv           map[string]string
+	ContainerImage         string
+	ContainerLabels        map[string]string
+	ContainerName          string
+	ContainerNetworks      map[string][]string
+	ContainerNetworkLabels map[string]map[string]string
+	ContainerPort          string
+	Driver                 string
+	DSN                    string
+	HostPort               string
+}
+
+// Validate the DatabaseOption struct, if all required atrributes are valid
+func (dbOptions *PostgresOptions) Validate() error {
+
+	// Required strings
+	for _, values := range [][]string{
+		0: {"ContainerImage", dbOptions.ContainerImage},
+		1: {"ContainerPort", dbOptions.ContainerPort},
+		2: {"Driver", dbOptions.Driver},
+		3: {"HostPort", dbOptions.HostPort},
+	} {
+		if len(values[1]) <= 0 {
+			return fmt.Errorf("%w: Attribute %v is empty", ErrInvalidAttr, values[0])
+		}
+	}
+
+	// Require initialized maps
+	for key, value := range map[string]interface{}{
+		"ContainerEnv":           dbOptions.ContainerEnv,
+		"ContainerLabels":        dbOptions.ContainerLabels,
+		"ContainerNetworks":      dbOptions.ContainerNetworks,
+		"ContainerNetworkLabels": dbOptions.ContainerNetworkLabels,
+	} {
+		if value == nil {
+			return fmt.Errorf("%w: Attribut %v is not initialized", ErrInvalidAttr, key)
+		}
+	}
+
+	// Required postgres environment variables
+	for _, key := range []string{
+		"POSTGRES_PASSWORD",
+		"POSTGRES_USER",
+		"POSTGRES_DB",
+	} {
+		if _, present := dbOptions.ContainerEnv[key]; !present {
+			return fmt.Errorf("%w: Required env %v not defined", ErrInvalidAttr, key)
+		}
+	}
+
+	// Supported drivers
+	found := false
+	for _, supportedDriver := range []string{
+		"postgres",
+	} {
+		if supportedDriver == dbOptions.Driver {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return fmt.Errorf("%w: Driver %v not supported", ErrInvalidAttr, dbOptions.Driver)
+	}
+
+	// Protect well-known ports
+	hostPort, err := strconv.ParseInt(dbOptions.HostPort, 10, 64)
+	if err != nil {
+		return fmt.Errorf("Failed to parse hostport %v: %w", dbOptions.HostPort, err)
+	}
+
+	if hostPort > 0 && hostPort < 1024 {
+		return fmt.Errorf("%w: ContainerPort %v: Protect well-known ports between 1-1023", ErrInvalidAttr, dbOptions.HostPort)
+	}
+
+	return nil
+}
+
+// NewPostgresDatabase starts a new postgres database based on default values
+func NewPostgresDatabase(t *testing.T) (*PostgresOptions, func()) {
+	return StartPostgres(t, NewPostgresOptions())
+}
+
+// NewPostgresOptions returns a new PostgresOptions structs with default values
+func NewPostgresOptions() *PostgresOptions {
+
+	return &PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "postgres",
+			"POSTGRES_USER":     "postgres",
+			"POSTGRES_DB":       "postgres",
+		},
+		ContainerImage:         "docker.io/library/postgres:13-alpine",
+		ContainerLabels:        make(map[string]string, 0),
+		ContainerName:          uuid.NewV4().String(),
+		ContainerNetworks:      make(map[string][]string, 0),
+		ContainerNetworkLabels: make(map[string]map[string]string, 0),
+		ContainerPort:          "5432",
+		Driver:                 "postgres",
+		DSN:                    "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
+		HostPort:               "0",
+	}
+}
+
+// StartPostgres starts a postgres container image based on the PostgresOption
+// struct
+func StartPostgres(t *testing.T, pgOptions *PostgresOptions) (*PostgresOptions, func()) {
+
+	var (
+		ctx     = context.Background()
+		require = require.New(t)
+
+		cleanupLabels = map[string]string{
+			uuid.NewV4().String(): uuid.NewV4().String(),
+		}
+	)
+
+	dockerClient, err := dockerutils.New()
+	require.NoError(err)
+
+	// Validate the PostgresOptions struct
+	require.NoError(pgOptions.Validate())
+
+	// Pre-Define DSN based on the pgOption attributes
+	pgOptions.DSN = fmt.Sprintf("postgres://%v:%v@localhost:%v/%v?sslmode=disable",
+		pgOptions.ContainerEnv["POSTGRES_USER"],
+		pgOptions.ContainerEnv["POSTGRES_PASSWORD"],
+		pgOptions.HostPort,
+		pgOptions.ContainerEnv["POSTGRES_DB"],
+	)
+
+	// Create missing networks
+	// The dockerutils package check only if the network exist and append the
+	// container with his aliasses as network endpoint. Additionally every network
+	// get his own network labels. If no network labels are defined, new labels
+	// will be generated. This is required to remove the networks by their labels.
+	networks, err := dockerClient.NetworkList(ctx, types.NetworkListOptions{})
+	require.NoError(err)
+
+	for networkName := range pgOptions.ContainerNetworks {
+		found := false
+		for _, network := range networks {
+			if network.Name == networkName {
+				found = true
+				break
+			}
+		}
+		if !found {
+			// Create network labels if there are no one defined
+			if _, present := pgOptions.ContainerNetworkLabels[networkName]; !present {
+				pgOptions.ContainerNetworkLabels[networkName] = make(map[string]string, 0)
+			}
+
+			// Append cleanup labels to the network
+			for key, value := range cleanupLabels {
+				pgOptions.ContainerNetworkLabels[networkName][key] = value
+			}
+
+			_, err := dockerClient.NetworkCreate(ctx, networkName, types.NetworkCreate{
+				Labels: pgOptions.ContainerNetworkLabels[networkName],
+			})
+			require.NoError(err)
+		}
+	}
+
+	// Append cleanup labels to the container
+	for key, value := range cleanupLabels {
+		pgOptions.ContainerLabels[key] = value
+	}
+
+	// Build database container
+	databaseContainerBuilder := dockerClient.NewBuilder(pgOptions.ContainerImage).
+		Env(pgOptions.ContainerEnv).
+		Labels(pgOptions.ContainerLabels).
+		Port(fmt.Sprintf("%s:%s", pgOptions.HostPort, pgOptions.ContainerPort)).
+		Pull().
+		WithName(pgOptions.ContainerName)
+
+	networkNames := make([]string, 0)
+	for networkName, aliasses := range pgOptions.ContainerNetworks {
+		if aliasses == nil {
+			aliasses = make([]string, 0)
+		}
+		if len(aliasses) <= 0 {
+			aliasses = append(aliasses, pgOptions.ContainerName)
+		}
+		databaseContainerBuilder.Network(networkName, aliasses...)
+		pgOptions.ContainerNetworks[networkName] = aliasses
+		networkNames = append(networkNames, networkName)
+	}
+
+	dbContainerID, err := databaseContainerBuilder.Start(ctx)
+	require.NoError(err)
+
+	// cleanupFunction to remove the database container with all defined networks.
+	// Skip network if the network has an additional endpoint
+	cleanupFunc := func() {
+		err := dockerClient.ContainerRemove(ctx, dbContainerID, types.ContainerRemoveOptions{Force: true})
+		require.NoError(err)
+
+		for networkName := range pgOptions.ContainerNetworks {
+			networks, err := dockerClient.NetworkListByLabels(ctx, pgOptions.ContainerNetworkLabels[networkName])
+			require.NoError(err)
+
+			for _, network := range networks {
+				if len(network.Containers) <= 0 {
+					err := dockerClient.NetworkRemove(ctx, network.ID)
+					require.NoError(err)
+				}
+			}
+		}
+	}
+
+	// Search for allocated port if public port is defined as random by 0
+	if pgOptions.HostPort == "0" {
+
+		containers, err := dockerClient.ContainerListByLabels(ctx, true, pgOptions.ContainerLabels)
+		require.NoError(err)
+
+		var pubPort string
+		for _, port := range containers[0].Ports {
+			if port.PrivatePort == 5432 {
+				pubPort = strconv.Itoa(int(port.PublicPort))
+				break
+			}
+		}
+		require.Greater(len(pubPort), 0, "Failed to detect allocated random port of the postgres container")
+
+		connectionURL, err := url.Parse(pgOptions.DSN)
+		require.NoError(err)
+
+		parts := strings.Split(connectionURL.Host, ":")
+		connectionURL.Host = fmt.Sprintf("%v:%v", parts[0], pubPort)
+		pgOptions.DSN = connectionURL.String()
+		pgOptions.HostPort = pubPort
+	}
+
+	var db *sql.DB
+	for i := 0; i < 10; i++ {
+		db, err = sql.Open(pgOptions.Driver, pgOptions.DSN)
+		if err != nil {
+			t.Log("Database not ready - wait 10 seconds")
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		err = db.Ping()
+		if err != nil {
+			t.Log("Database not ready - wait 10 seconds")
+			db.Close()
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(err)
+
+	err = db.Close()
+	require.NoError(err)
+
+	return pgOptions, cleanupFunc
+}

pkg/testutils/database/database_test.go

@@ -0,0 +1,331 @@
+package database_test
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"testing"
+
+	"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/database"
+	"git.cryptic.systems/volker.raschek/flucky/pkg/testutils/dockerutils"
+	uuid "github.com/satori/go.uuid"
+	"github.com/stretchr/testify/require"
+
+	_ "github.com/lib/pq"
+)
+
+func requireCountNetworks(ctx context.Context, dockerClient *dockerutils.Client, dbOptions *database.PostgresOptions, equal int, t *testing.T) {
+	require := require.New(t)
+	containers, err := dockerClient.ContainerListByLabels(ctx, true, dbOptions.ContainerLabels)
+	require.NoError(err)
+	require.Equal(equal, len(containers[0].NetworkSettings.Networks))
+}
+
+func requireContainerHasNetworksAndAliasses(ctx context.Context, dockerClient *dockerutils.Client, dbOptions *database.PostgresOptions, t *testing.T) {
+	require := require.New(t)
+
+	containers, err := dockerClient.ContainerListByNames(ctx, true, dbOptions.ContainerName)
+	require.NoError(err)
+	require.Contains(containers[0].Names, fmt.Sprintf("/%v", dbOptions.ContainerName))
+
+	for networkName, _ := range dbOptions.ContainerNetworks {
+		networks, err := dockerClient.NetworkListByNames(ctx, networkName)
+		require.NoError(err)
+		require.Equal(networkName, networks[0].Name)
+		require.Equal(dbOptions.ContainerName, networks[0].Containers[containers[0].ID].Name)
+	}
+}
+
+func requireTestCleanup(ctx context.Context, dockerClient *dockerutils.Client, cleanupLabels map[string]string, t *testing.T) {
+	require := require.New(t)
+	containers, err := dockerClient.ContainerListByLabels(ctx, true, cleanupLabels)
+	require.NoError(err)
+	require.NotNil(containers)
+	require.Len(containers, 0)
+
+	networks, err := dockerClient.NetworkListByLabels(ctx, cleanupLabels)
+	require.NoError(err)
+	require.NotNil(networks)
+	require.Len(networks, 0)
+}
+
+func TestPostgresDatabase(t *testing.T) {
+	require := require.New(t)
+	dbOptions, cleanup := database.NewPostgresDatabase(t)
+	t.Cleanup(func() { cleanup() })
+
+	require.NotEqual(0, dbOptions.HostPort)
+
+	dockerClient, err := dockerutils.New()
+	require.NoError(err)
+
+	dbContainers, err := dockerClient.ContainerListByLabels(context.Background(), true, dbOptions.ContainerLabels)
+	require.NoError(err)
+	require.NotNil(dbContainers)
+	require.Len(dbContainers, 1)
+
+	dbo, err := sql.Open(dbOptions.Driver, dbOptions.DSN)
+	require.NoError(err)
+	require.NoError(dbo.Ping())
+
+	query := "SELECT 1"
+	row := dbo.QueryRow(query)
+
+	var i int
+	err = row.Scan(&i)
+	require.NoError(err)
+	require.Equal(1, i)
+}
+
+func TestPostgresNetwork(t *testing.T) {
+	var (
+		ctx     = context.Background()
+		require = require.New(t)
+
+		cleanupLabels = map[string]string{
+			uuid.NewV4().String(): uuid.NewV4().String(),
+		}
+	)
+
+	dockerClient, err := dockerutils.New()
+	require.NoError(err)
+	t.Cleanup(func() {
+		dockerClient.ContainerRemoveByLabels(ctx, cleanupLabels)
+		dockerClient.NetworkRemoveByLabels(ctx, cleanupLabels)
+		dockerClient.Close()
+	})
+
+	// TestDefaultNetwork
+	dbOptionsPointer, cleanup := database.NewPostgresDatabase(t)
+	t.Cleanup(cleanup)
+
+	require.NotNil(dbOptionsPointer.ContainerNetworks)
+	require.Len(dbOptionsPointer.ContainerNetworks, 0)
+	requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptionsPointer, t)
+
+	// TestRandomNetwork
+	dbOptions := database.NewPostgresOptions()
+	dbOptions.HostPort = "0"
+	dbOptions.ContainerName = "Volker"
+	dbOptions.ContainerLabels = cleanupLabels
+	dbOptions.ContainerNetworks[uuid.NewV4().String()] = []string{
+		uuid.NewV4().String(),
+		uuid.NewV4().String(),
+	}
+	_, cleanup = database.StartPostgres(t, dbOptions)
+	t.Cleanup(cleanup)
+
+	requireCountNetworks(ctx, dockerClient, dbOptions, 1, t)
+	requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptions, t)
+
+	// TestRandomMultipleNetworks
+	containerNetworkNameA := uuid.NewV4().String()
+	containerNetworkNameB := uuid.NewV4().String()
+	containerNetworks := map[string][]string{
+		containerNetworkNameA: {
+			uuid.NewV4().String(),
+			uuid.NewV4().String(),
+		},
+		containerNetworkNameB: {
+			uuid.NewV4().String(),
+			uuid.NewV4().String(),
+		},
+	}
+
+	containerNetworkLabels := map[string]map[string]string{
+		containerNetworkNameA: cleanupLabels,
+		containerNetworkNameB: cleanupLabels,
+	}
+
+	dbOptions = database.NewPostgresOptions()
+	dbOptions.HostPort = "0"
+	dbOptions.ContainerName = "Raschek"
+	dbOptions.ContainerNetworks = containerNetworks
+	dbOptions.ContainerNetworkLabels = containerNetworkLabels
+	dbOptions.ContainerLabels = cleanupLabels
+
+	_, cleanup = database.StartPostgres(t, dbOptions)
+	t.Cleanup(cleanup)
+
+	requireCountNetworks(ctx, dockerClient, dbOptions, 2, t)
+	requireContainerHasNetworksAndAliasses(ctx, dockerClient, dbOptions, t)
+
+	// TestCleanup
+	// Test if a network with an additional endpoint will not be removed
+	containerNetwork := uuid.NewV4().String()
+	containerNetworkA := map[string][]string{
+		containerNetwork: {
+			uuid.NewV4().String(),
+			uuid.NewV4().String(),
+		},
+	}
+	containerNetworkB := map[string][]string{
+		containerNetwork: {
+			uuid.NewV4().String(),
+			uuid.NewV4().String(),
+		},
+	}
+	containerNetworkLabels = map[string]map[string]string{
+		containerNetwork: cleanupLabels,
+	}
+
+	dbOptions = database.NewPostgresOptions()
+	dbOptions.HostPort = "0"
+	dbOptions.ContainerName = "is"
+	dbOptions.ContainerLabels = cleanupLabels
+	dbOptions.ContainerNetworks = containerNetworkA
+	dbOptions.ContainerNetworkLabels = containerNetworkLabels
+	_, cleanup = database.StartPostgres(t, dbOptions)
+
+	dbOptions = database.NewPostgresOptions()
+	dbOptions.HostPort = "0"
+	dbOptions.ContainerName = "the_best"
+	dbOptions.ContainerLabels = cleanupLabels
+	dbOptions.ContainerNetworks = containerNetworkB
+	dbOptions.ContainerNetworkLabels = containerNetworkLabels
+	_, cleanupSecond := database.StartPostgres(t, dbOptions)
+
+	cleanup()
+
+	networkResources, err := dockerClient.NetworkListByNames(ctx, containerNetwork)
+	require.NoError(err)
+	require.Len(networkResources[0].Containers, 1)
+
+	cleanupSecond()
+	requireTestCleanup(ctx, dockerClient, cleanupLabels, t)
+}
+
+func TestPostgresOptionsValidate(t *testing.T) {
+
+	require := require.New(t)
+
+	// ContainerImage is empty
+	dbOptions := database.PostgresOptions{}
+	err := dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Attribute ContainerImage is empty")
+
+	// ContainerPort is empty
+	dbOptions = database.PostgresOptions{
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Attribute ContainerPort is empty")
+
+	// Driver is empty
+	dbOptions = database.PostgresOptions{
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Attribute Driver is empty")
+
+	// HostPort is empty
+	dbOptions = database.PostgresOptions{
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "sdfsdfsdf",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Attribute HostPort is empty")
+
+	// ContainerEnv: POSTGRES_PASSWORD
+	dbOptions = database.PostgresOptions{
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "sdfsdfsdf",
+		HostPort:       "asdasd",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_PASSWORD not defined")
+
+	// ContainerEnv: POSTGRES_USER
+	dbOptions = database.PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "HelloWorld",
+		},
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "sdfsdfsdf",
+		HostPort:       "asdasd",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_USER not defined")
+
+	// ContainerEnv: POSTGRES_USER
+	dbOptions = database.PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "postgres",
+			"POSTGRES_USER":     "postgres",
+		},
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "sdfsdfsdf",
+		HostPort:       "asdasd",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Required env POSTGRES_DB not defined")
+
+	// Driver not supported
+	dbOptions = database.PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "postgres",
+			"POSTGRES_USER":     "postgres",
+			"POSTGRES_DB":       "postgres",
+		},
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "sdfsdfsdf",
+		HostPort:       "asdasd",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.True(errors.Is(err, database.ErrInvalidAttr))
+	require.EqualError(err, "Invalid DatabaseOption attribute: Driver sdfsdfsdf not supported")
+
+	// HostPort: Failed to parse
+	dbOptions = database.PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "postgres",
+			"POSTGRES_USER":     "postgres",
+			"POSTGRES_DB":       "postgres",
+		},
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "postgres",
+		HostPort:       "asdasd",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.EqualError(err, "Failed to parse hostport asdasd: strconv.ParseInt: parsing \"asdasd\": invalid syntax")
+
+	// HostPort: Well-Known port
+	dbOptions = database.PostgresOptions{
+		ContainerEnv: map[string]string{
+			"POSTGRES_PASSWORD": "postgres",
+			"POSTGRES_USER":     "postgres",
+			"POSTGRES_DB":       "postgres",
+		},
+		ContainerImage: "docker.io/library/postgres:13-alpine",
+		ContainerPort:  "5432",
+		Driver:         "postgres",
+		HostPort:       "443",
+	}
+	err = dbOptions.Validate()
+	require.Error(err)
+	require.EqualError(err, "Invalid DatabaseOption attribute: ContainerPort 443: Protect well-known ports between 1-1023")
+}

pkg/testutils/dockerutils/client.go

@@ -31,7 +31,8 @@ func (client *Client) Close() error {
 	if client.watcher != nil {
 		client.watcher.stop()
 	}
-	return client.Close()
+
+	return client.Client.Close()
 }
 
 // ContainerListByLabels returns only containers which match by given labels