From d94b42e0725f2fb8506e5015bd32225306ac3061 Mon Sep 17 00:00:00 2001 From: Markus Pesch Date: Wed, 26 Jul 2023 09:10:11 +0200 Subject: [PATCH] fix(ci): Makefile, Containerfile --- .containerignore | 1 + .gitignore | 1 + Containerfile | 20 +++++++ Makefile | 114 +++++++++++++++++++++++++++++++++++++++ README.md | 118 +++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 72 +++++++++++++++++++++++++ go.mod | 10 +++- go.sum | 10 ++++ main.go | 7 ++- pkg/fetcher/fetcher.go | 99 ++++++++++++++++++++++++++++++++++ 10 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 .containerignore create mode 100644 .gitignore create mode 100644 Containerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 pkg/fetcher/fetcher.go diff --git a/.containerignore b/.containerignore new file mode 100644 index 0000000..86f582a --- /dev/null +++ b/.containerignore @@ -0,0 +1 @@ +dcmerge \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86f582a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dcmerge \ No newline at end of file diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..82cd1cf --- /dev/null +++ b/Containerfile @@ -0,0 +1,20 @@ +FROM docker.io/library/golang:1.20.6-alpine3.18 AS build + +RUN apk add git make + +WORKDIR /workspace +ADD ./ /workspace + +RUN make install \ + DESTDIR=/cache \ + PREFIX=/usr \ + VERSION=${VERSION} + +FROM docker.io/library/alpine:3.18.2 + +COPY --from=build /cache / + +WORKDIR /workspace +VOLUME [ "/workspace" ] + +ENTRYPOINT [ "/usr/bin/dcmerge" ] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c5bff2a --- /dev/null +++ b/Makefile @@ -0,0 +1,114 @@ +EXECUTABLE=dcmerge +VERSION?=$(shell git describe --abbrev=0)+hash.$(shell git rev-parse --short HEAD) + +# Destination directory and prefix to place the compiled binaries, documentaions +# and other files. +DESTDIR?= +PREFIX?=/usr/local + +# CONTAINER_RUNTIME +# The CONTAINER_RUNTIME variable will be used to specified the path to a +# container runtime. This is needed to start and run a container image. +CONTAINER_RUNTIME?=$(shell which podman) + +# DCMERGE_IMAGE_REGISTRY_NAME +# Defines the name of the new container to be built using several variables. +DCMERGE_IMAGE_REGISTRY_NAME:=git.cryptic.systems +DCMERGE_IMAGE_REGISTRY_USER:=volker.raschek + +DCMERGE_IMAGE_NAMESPACE?=${DCMERGE_IMAGE_REGISTRY_USER} +DCMERGE_IMAGE_NAME:=${EXECUTABLE} +DCMERGE_IMAGE_VERSION?=latest +DCMERGE_IMAGE_FULLY_QUALIFIED=${DCMERGE_IMAGE_REGISTRY_NAME}/${DCMERGE_IMAGE_NAMESPACE}/${DCMERGE_IMAGE_NAME}:${DCMERGE_IMAGE_VERSION} +DCMERGE_IMAGE_UNQUALIFIED=${DCMERGE_IMAGE_NAMESPACE}/${DCMERGE_IMAGE_NAME}:${DCMERGE_IMAGE_VERSION} + +# BIN +# ============================================================================== +dcmerge: + CGO_ENABLED=0 \ + GOPRIVATE=$(shell go env GOPRIVATE) \ + GOPROXY=$(shell go env GOPROXY) \ + GONOPROXY=$(shell go env GONOPROXY) \ + GONOSUMDB=$(shell go env GONOSUMDB) \ + GOSUMDB=$(shell go env GOSUMDB) \ + go build -ldflags "-X 'main.version=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" -o ${@} main.go + +# CLEAN +# ============================================================================== +PHONY+=clean +clean: + rm --force --recursive dcmerge + +# TESTS +# ============================================================================== +PHONY+=test/unit +test/unit: + go test -v -p 1 -coverprofile=coverage.txt -covermode=count -timeout 1200s ./pkg/... + +PHONY+=test/integration +test/integration: + go test -v -p 1 -count=1 -timeout 1200s ./it/... + +PHONY+=test/coverage +test/coverage: test/unit + go tool cover -html=coverage.txt + +# GOLANGCI-LINT +# ============================================================================== +PHONY+=golangci-lint +golangci-lint: + golangci-lint run --concurrency=$(shell nproc) + +# INSTALL +# ============================================================================== +PHONY+=uninstall +install: dcmerge + install --directory ${DESTDIR}/etc/bash_completion.d + ./dcmerge completion bash > ${DESTDIR}/etc/bash_completion.d/${EXECUTABLE} + + install --directory ${DESTDIR}${PREFIX}/bin + install --mode 0755 ${EXECUTABLE} ${DESTDIR}${PREFIX}/bin/${EXECUTABLE} + + install --directory ${DESTDIR}${PREFIX}/share/licenses/${EXECUTABLE} + install --mode 0644 LICENSE ${DESTDIR}${PREFIX}/share/licenses/${EXECUTABLE}/LICENSE + +# UNINSTALL +# ============================================================================== +PHONY+=uninstall +uninstall: + -rm --force --recursive \ + ${DESTDIR}/etc/bash_completion.d/${EXECUTABLE} \ + ${DESTDIR}${PREFIX}/bin/${EXECUTABLE} \ + ${DESTDIR}${PREFIX}/share/licenses/${EXECUTABLE} + +# BUILD CONTAINER IMAGE +# ============================================================================== +PHONY+=container-image/build +container-image/build: + ${CONTAINER_RUNTIME} build \ + --build-arg VERSION=${VERSION} \ + --file Containerfile \ + --no-cache \ + --pull \ + --tag ${DCMERGE_IMAGE_FULLY_QUALIFIED} \ + --tag ${DCMERGE_IMAGE_UNQUALIFIED} \ + . + +# DELETE CONTAINER IMAGE +# ============================================================================== +PHONY:=container-image/delete +container-image/delete: + - ${CONTAINER_RUNTIME} image rm ${DCMERGE_IMAGE_FULLY_QUALIFIED} ${DCMERGE_IMAGE_UNQUALIFIED} + +# PUSH CONTAINER IMAGE +# ============================================================================== +PHONY+=container-image/push +container-image/push: + echo ${DCMERGE_IMAGE_REGISTRY_PASSWORD} | ${CONTAINER_RUNTIME} login ${DCMERGE_IMAGE_REGISTRY_NAME} --username ${DCMERGE_IMAGE_REGISTRY_USER} --password-stdin + ${CONTAINER_RUNTIME} push ${DCMERGE_IMAGE_FULLY_QUALIFIED} + +# PHONY +# ============================================================================== +# Declare the contents of the PHONY variable as phony. We keep that information +# in a variable so we can use it in if_changed. +.PHONY: ${PHONY} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..730f73b --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +# dcmerge + +`dcmerge` is a small program to merge docker-compose files from multiple +sources. The following source are currently supported: + +- File +- HTTP/HTTPS + +Furthermore, dcmerge support different ways to merge multiple docker-compose +files. + +- The default merge, add missing secrets, services, networks and volumes. +- The existing-win merge, add and protect existing attributes. +- The last-win merge, add or overwrite existing attributes. + +## default + +Merge only missing secrets, services, networks and volumes without respecting +their attributes. For example, when the service `app` is already declared, it is +not possible to add the service `app` twice. The second service will be +completely skipped. + +```yaml +--- +# cat ~/docker-compose-A.yaml +services: + app: + environments: + - CLIENT_SECRET=HelloWorld123 + image: example.local/app/name:0.1.0 +--- +# cat ~/docker-compose-B.yaml +services: + app: + image: app/name:2.3.0 + volume: + - /etc/localtime:/etc/localtime + - /dev/urandom:/etc/urandom + db: + image: postgres + volume: + - /etc/localtime:/etc/localtime + - /dev/urandom:/etc/urandom +--- +# dcmerge ~/docker-compose-A.yaml ~/docker-compose-B.yaml +services: + app: + environments: + - CLIENT_SECRET=HelloWorld123 + image: example.local/app/name:0.1.0 + db: + image: postgres + volume: + - /etc/localtime:/etc/localtime + - /dev/urandom:/etc/urandom +``` + +## existing-win + +The existing-win merge protects existing attributes. For example there are two +different docker-compose files, but booth has the same environment variable +`CLIENT_SECRET` defined with different values. The first declaration of the +attribute wins and is for overwriting protected. + +```yaml +--- +# cat ~/docker-compose-A.yaml +services: + app: + environments: + - CLIENT_SECRET=HelloWorld123 + image: example.local/app/name:0.1.0 +--- +# cat ~/docker-compose-B.yaml +services: + app: + environments: + - CLIENT_SECRET=FooBar123 + image: example.local/app/name:0.1.0 +--- +# dcmerge --existing-win ~/docker-compose-A.yaml ~/docker-compose-B.yaml +services: + app: + environments: + - CLIENT_SECRET=HelloWorld123 + image: example.local/app/name:0.1.0 +``` + +## last-win + +The last-win merge overwrite recursive existing attributes. For example there +are two different docker-compose files, but booth has the same environment +variable `CLIENT_SECRET` defined with different values. The last passed +docker-compose file which contains this environment wins. + +```yaml +--- +# cat ~/docker-compose-A.yaml +services: + app: + environments: + - CLIENT_SECRET=HelloWorld123 + image: example.local/app/name:0.1.0 +--- +# cat ~/docker-compose-B.yaml +services: + app: + environments: + - CLIENT_SECRET=FooBar123 + image: example.local/app/name:0.1.0 +--- +# dcmerge --last-win ~/docker-compose-A.yaml ~/docker-compose-B.yaml +services: + app: + environments: + - CLIENT_SECRET=FooBar123 + image: example.local/app/name:0.1.0 +``` diff --git a/cmd/root.go b/cmd/root.go index 1d619dd..43e5e91 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1 +1,73 @@ package cmd + +import ( + "fmt" + "os" + + "git.cryptic.systems/volker.raschek/dcmerge/pkg/domain/dockerCompose" + "git.cryptic.systems/volker.raschek/dcmerge/pkg/fetcher" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +func Execute(version string) error { + completionCmd := &cobra.Command{ + Use: "completion [bash|zsh|fish|powershell]", + Short: "Generate completion script", + Long: "To load completions", + DisableFlagsInUseLine: true, + ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, + Args: cobra.ExactValidArgs(1), + Run: func(cmd *cobra.Command, args []string) { + switch args[0] { + case "bash": + cmd.Root().GenBashCompletion(os.Stdout) + case "zsh": + cmd.Root().GenZshCompletion(os.Stdout) + case "fish": + cmd.Root().GenFishCompletion(os.Stdout, true) + case "powershell": + cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout) + } + }, + } + + rootCmd := &cobra.Command{ + Use: "dcmerge", + Args: cobra.MinimumNArgs(1), + Short: "Merge docker-compose files from multiple resources", + Example: `dcmerge docker-compose.yml ./integration-test/docker-compose.yml +dcmerge docker-compose.yml https://git.example.local/user/repo/docker-compose.yml`, + RunE: run, + Version: version, + } + rootCmd.Flags().BoolP("merge-last-win", "l", true, "Overwrite existing attributes") + rootCmd.AddCommand(completionCmd) + + return rootCmd.Execute() +} + +func run(cmd *cobra.Command, args []string) error { + mergeLastWin, err := cmd.Flags().GetBool("merge-last-win") + if err != nil { + return fmt.Errorf("Failed to parse flag merge-last-win: %s", err) + } + + dockerComposeConfig := dockerCompose.NewConfig() + + dockerComposeConfigs, err := fetcher.Fetch(args...) + if err != nil { + return err + } + + for _, config := range dockerComposeConfigs { + switch { + case mergeLastWin: + dockerComposeConfig.MergeLastWin(config) + } + + } + + yamlEncoder := yaml.NewEncoder(os.Stdout) + return yamlEncoder.Encode(dockerComposeConfig) +} diff --git a/go.mod b/go.mod index 6d48064..be3e42f 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,16 @@ module git.cryptic.systems/volker.raschek/dcmerge go 1.20 -require github.com/stretchr/testify v1.8.4 +require ( + github.com/spf13/cobra v1.7.0 + github.com/stretchr/testify v1.8.4 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index fa4b6e6..44aa0ef 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,20 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 081abb2..2a26e69 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,10 @@ package main -import "log" +import "git.cryptic.systems/volker.raschek/dcmerge/cmd" + +var buildDate string +var version string func main() { - log.Println("Hello World") + cmd.Execute(version) } diff --git a/pkg/fetcher/fetcher.go b/pkg/fetcher/fetcher.go new file mode 100644 index 0000000..fd69792 --- /dev/null +++ b/pkg/fetcher/fetcher.go @@ -0,0 +1,99 @@ +package fetcher + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "os" + + "git.cryptic.systems/volker.raschek/dcmerge/pkg/domain/dockerCompose" + "gopkg.in/yaml.v3" +) + +func Fetch(urls ...string) ([]*dockerCompose.Config, error) { + dockerComposeConfigs := make([]*dockerCompose.Config, 0) + + for _, rawURL := range urls { + dockerComposeURL, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + + switch { + case dockerComposeURL.Scheme == "http" || dockerComposeURL.Scheme == "https": + dockerComposeConfig, err := getDockerComposeViaHTTP(dockerComposeURL.String()) + if err != nil { + return nil, err + } + + dockerComposeConfigs = append(dockerComposeConfigs, dockerComposeConfig) + case dockerComposeURL.Scheme == "file": + fallthrough + default: + dockerComposeConfig, err := readDockerComposeFromFile(dockerComposeURL.Path) + if err != nil { + return nil, err + } + + dockerComposeConfigs = append(dockerComposeConfigs, dockerComposeConfig) + } + } + + return dockerComposeConfigs, nil +} + +var ErrorPathIsDir error = errors.New("Path is a directory") + +func getDockerComposeViaHTTP(url string) (*dockerCompose.Config, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Received unexpected HTTP-Statuscode %v", resp.StatusCode) + } + + dockerCompose := dockerCompose.NewConfig() + + yamlDecoder := yaml.NewDecoder(resp.Body) + err = yamlDecoder.Decode(&dockerCompose) + if err != nil { + return nil, err + } + + return dockerCompose, nil +} + +func readDockerComposeFromFile(name string) (*dockerCompose.Config, error) { + fileStat, err := os.Stat(name) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, err + case fileStat.IsDir(): + return nil, fmt.Errorf("%w: %s", ErrorPathIsDir, name) + } + + file, err := os.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + + dockerCompose := dockerCompose.NewConfig() + + yamlDecoder := yaml.NewDecoder(file) + err = yamlDecoder.Decode(&dockerCompose) + if err != nil { + return nil, err + } + + return dockerCompose, nil +}