Merge branch 'remove-fail2ban-database-based-metrics' into 'main'

Remove fail2ban database based metrics

See merge request hectorjsmith/fail2ban-prometheus-exporter!43

README.md

@@ -15,21 +15,10 @@ Once the exporter is running, metrics are available at `localhost:9191/metrics`.
 
 (The default port is `9191` but can be modified with the `-port` flag)
 
-### 1.1. Socket
+The exporter communicates with the fail2ban server over its socket.
-The recommended way to run the exporter is to point it at the fail2ban server socket.
+This allows the data collected by the exporter to always align with the output of the `fail2ban-client`.
-This allows the exporter to communicate with the server the same way `fail2ban-client` does and ensures the metrics it collects align with the values reported by `fail2ban-client status <jail>`.
 
-The default path to the socket is: `/var/run/fail2ban/fail2ban.sock`
+The default location of the socket is: `/var/run/fail2ban/fail2ban.sock`
-
-### 1.2. Deprecated: Database
-The original way to collect metrics is to read them from the fail2ban database.
-This has now been deprecated in favour of using the socket.
-The reason being that database metrics do not always align with the output of `fail2ban-client status <jail>` and cause confusion.
-See [#11](https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter/-/issues/11) for more details.
-
-If necessary, these metrics can still be exported by providing the database path to the exporter.
-
-The default path to the fail2ban database is: `/var/lib/fail2ban/fail2ban.sqlite3`
 
 ## 2. Running the Exporter
 
@@ -44,8 +33,6 @@ See the [releases page](https://gitlab.com/hectorjsmith/fail2ban-prometheus-expo
 ```
 $ fail2ban-prometheus-exporter -h
 
-  -db string
-        path to the fail2ban sqlite database (deprecated)
   -port int
         port to use for the metrics server (default 9191)
   -socket string
@@ -64,7 +51,7 @@ $ fail2ban-prometheus-exporter -h
 fail2ban-prometheus-exporter -socket /var/run/fail2ban/fail2ban.sock -port 9191
 ```
 
-Note that the exporter will need read access to the fail2ban socket or database.
+Note that the exporter will need read access to the fail2ban socket.
 
 ### 2.1. Compile from Source
 
@@ -75,7 +62,7 @@ Run `go mod download` to download all necessary dependencies before running the
 
 ## 3. Running in Docker
 
-If use of docker is desired, an official docker image is available on the Gitlab container registry.
+An official docker image is available on the Gitlab container registry.
 Use it by pulling the following image:
 
 ```
@@ -87,11 +74,10 @@ See the [registry page](https://gitlab.com/hectorjsmith/fail2ban-prometheus-expo
 
 ### 3.1. Volumes
 
-The docker image is designed to run by mounting either the fail2ban sqlite3 database of the fail2ban run folder.
+The docker image is designed to run by mounting the fail2ban run folder.
-- The database should be mounted at: `/app/fail2ban.sqlite3`
+The run folder should be mounted in the container at: `/var/run/fail2ban`.
-- The run folder should be mounted at: `/var/run/fail2ban`
 
-Both paths can be mounted with readonly (`ro`) permissions.
+The folder can be mounted with read-only (`ro`) permissions.
 
 **NOTE:** While it is possible to mount the `fail2ban.sock` file directly, it is recommended to mount the parent folder instead.
 The `.sock` file is deleted by fail2ban on shutdown and re-created on startup and this causes problems for the docker mount.
@@ -104,7 +90,6 @@ Use the following command to run the exporter as a docker container.
 ```
 docker run -d \
     --name "fail2ban-exporter" \
-    -v /var/lib/fail2ban/fail2ban.sqlite3:/app/fail2ban.sqlite3:ro \
     -v /var/run/fail2ban:/var/run/fail2ban:ro \
     -p "9191:9191" \
     registry.gitlab.com/hectorjsmith/fail2ban-prometheus-exporter:latest
@@ -120,7 +105,6 @@ services:
   exporter:
     image: registry.gitlab.com/hectorjsmith/fail2ban-prometheus-exporter:latest
     volumes:
-    - /var/lib/fail2ban/fail2ban.sqlite3:/app/fail2ban.sqlite3:ro
     - /var/run/fail2ban/:/var/run/fail2ban:ro
     ports:
     - "9191:9191"
@@ -221,50 +205,7 @@ Status for the jail: sshd|- Filter
    `- Banned IP list:   ...
 ```
 
-### 4.2. Database Metrics (deprecated)
+### 4.2. Textfile Metrics
-
-These are the original metrics exported by the initial release of the exporter.
-They are all based on the data stored in the fail2ban sqlite3 database.
-
-*These metrics are deprecated and will be removed in a future release.*
-
-All metrics are prefixed with `fail2ban_`.
-
-Exposed metrics:
-* `up` - Returns 1 if the service is up
-* `errors` - Returns the number of errors found since startup
-* `enabled_jails` - Returns 1 for each jail that is enabled, 0 if disabled.
-* `bad_ips` (per jail)
-    * A *bad IP* is defined as an IP that has been banned at least once in the past
-    * Bad IPs are counted per jail
-* `banned_ips` (per jail)
-    * A *banned IP* is defined as an IP that is currently banned on the firewall
-    * Banned IPs are counted per jail
-
-**Sample**
-
-```
-# HELP fail2ban_bad_ips (Deprecated) Number of bad IPs stored in the database (per jail).
-# TYPE fail2ban_bad_ips gauge
-fail2ban_bad_ips{jail="recidive"} 0
-fail2ban_bad_ips{jail="sshd"} 0
-# HELP fail2ban_banned_ips (Deprecated) Number of banned IPs stored in the database (per jail).
-# TYPE fail2ban_banned_ips gauge
-fail2ban_banned_ips{jail="recidive"} 0
-fail2ban_banned_ips{jail="sshd"} 0
-# HELP fail2ban_enabled_jails (Deprecated) Enabled jails.
-# TYPE fail2ban_enabled_jails gauge
-fail2ban_enabled_jails{jail="recidive"} 1
-fail2ban_enabled_jails{jail="sshd"} 1
-# HELP fail2ban_errors (Deprecated) Number of errors found since startup.
-# TYPE fail2ban_errors counter
-fail2ban_errors{type="db"} 0
-# HELP fail2ban_up (Deprecated) Was the last fail2ban query successful.
-# TYPE fail2ban_up gauge
-fail2ban_up 1
-```
-
-### 4.3. Textfile Metrics
 
 For more flexibility the exporter also allows exporting metrics collected from a text file.
 

docker/run.sh

@@ -3,18 +3,11 @@
 # Print version to logs for debugging purposes
 /app/fail2ban-prometheus-exporter -version
 
-db_path=/app/fail2ban.sqlite3
 socket_path=/var/run/fail2ban/fail2ban.sock
 textfile_dir=/app/textfile/
 textfile_enabled=false
 
-# Blank out the file paths if they do not exist - a hacky way to only use these files if they were mounted into the container.
+# Enable textfile metrics if the folder exists (i.e. was mounted by docker)
-if [ ! -f "$db_path" ]; then
-    db_path=""
-fi
-if [ ! -S "$socket_path" ]; then
-    socket_path=""
-fi
 if [ -d $textfile_dir ]; then
     textfile_enabled=true
 fi
@@ -22,7 +15,6 @@ fi
 # Start the exporter (use exec to support graceful shutdown)
 # Inspired by: https://akomljen.com/stopping-docker-containers-gracefully/
 exec /app/fail2ban-prometheus-exporter \
-    -db "$db_path" \
     -socket "$socket_path" \
     -collector.textfile=$textfile_enabled \
     -collector.textfile.directory="$textfile_dir"

src/cfg/cfg.go

@@ -14,7 +14,6 @@ const (
 type AppSettings struct {
 	VersionMode          bool
 	MetricsPort          int
-	Fail2BanDbPath       string
 	Fail2BanSocketPath   string
 	FileCollectorPath    string
 	FileCollectorEnabled bool
@@ -24,11 +23,13 @@ func Parse() *AppSettings {
 	appSettings := &AppSettings{}
 	flag.BoolVar(&appSettings.VersionMode, "version", false, "show version info and exit")
 	flag.IntVar(&appSettings.MetricsPort, "port", 9191, "port to use for the metrics server")
-	flag.StringVar(&appSettings.Fail2BanDbPath, "db", "", "path to the fail2ban sqlite database (deprecated)")
 	flag.StringVar(&appSettings.Fail2BanSocketPath, "socket", "", "path to the fail2ban server socket")
 	flag.BoolVar(&appSettings.FileCollectorEnabled, "collector.textfile", false, "enable the textfile collector")
 	flag.StringVar(&appSettings.FileCollectorPath, "collector.textfile.directory", "", "directory to read text files with metrics from")
 
+	// deprecated: to be removed in next version
+	_ = flag.String("db", "", "path to the fail2ban sqlite database (removed)")
+
 	flag.Parse()
 	appSettings.validateFlags()
 	return appSettings
@@ -37,8 +38,8 @@ func Parse() *AppSettings {
 func (settings *AppSettings) validateFlags() {
 	var flagsValid = true
 	if !settings.VersionMode {
-		if settings.Fail2BanDbPath == "" && settings.Fail2BanSocketPath == "" {
+		if settings.Fail2BanSocketPath == "" {
-			fmt.Println("at least one of the following flags must be provided: 'db', 'socket'")
+			fmt.Println("fail2ban socket path must not be blank")
 			flagsValid = false
 		}
 		if settings.MetricsPort < minServerPort || settings.MetricsPort > maxServerPort {

src/collector/f2b/collector.go

@@ -2,14 +2,12 @@ package f2b
 
 import (
 	"fail2ban-prometheus-exporter/cfg"
-	fail2banDb "fail2ban-prometheus-exporter/db"
 	"fail2ban-prometheus-exporter/socket"
 	"github.com/prometheus/client_golang/prometheus"
 	"log"
 )
 
 type Collector struct {
-	db                         *fail2banDb.Fail2BanDB
 	socketPath                 string
 	exporterVersion            string
 	lastError                  error
@@ -19,65 +17,39 @@ type Collector struct {
 }
 
 func NewExporter(appSettings *cfg.AppSettings, exporterVersion string) *Collector {
-	colector := &Collector{
+	return &Collector{
+		socketPath:                 appSettings.Fail2BanSocketPath,
 		exporterVersion:            exporterVersion,
 		lastError:                  nil,
 		dbErrorCount:               0,
 		socketConnectionErrorCount: 0,
 		socketRequestErrorCount:    0,
 	}
-	if appSettings.Fail2BanDbPath != "" {
-		log.Print("database-based metrics have been deprecated and will be removed in a future release")
-		colector.db = fail2banDb.MustConnectToDb(appSettings.Fail2BanDbPath)
-	}
-	if appSettings.Fail2BanSocketPath != "" {
-		colector.socketPath = appSettings.Fail2BanSocketPath
-	}
-	return colector
 }
 
 func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
-	if c.db != nil {
+	ch <- metricServerUp
-		ch <- deprecatedMetricUp
+	ch <- metricJailCount
-		ch <- deprecatedMetricBadIpsPerJail
+	ch <- metricJailFailedCurrent
-		ch <- deprecatedMetricBannedIpsPerJail
+	ch <- metricJailFailedTotal
-		ch <- deprecatedMetricEnabledJails
+	ch <- metricJailBannedCurrent
-		ch <- deprecatedMetricErrorCount
+	ch <- metricJailBannedTotal
-	}
-	if c.socketPath != "" {
-		ch <- metricServerUp
-		ch <- metricJailCount
-		ch <- metricJailFailedCurrent
-		ch <- metricJailFailedTotal
-		ch <- metricJailBannedCurrent
-		ch <- metricJailBannedTotal
-	}
 	ch <- metricErrorCount
 }
 
 func (c *Collector) Collect(ch chan<- prometheus.Metric) {
-	if c.db != nil {
+	s, err := socket.ConnectToSocket(c.socketPath)
-		c.collectDeprecatedBadIpsPerJailMetrics(ch)
+	if err != nil {
-		c.collectDeprecatedBannedIpsPerJailMetrics(ch)
+		log.Printf("error opening socket: %v", err)
-		c.collectDeprecatedEnabledJailMetrics(ch)
+		c.socketConnectionErrorCount++
-		c.collectDeprecatedUpMetric(ch)
-		c.collectDeprecatedErrorCountMetric(ch)
-	}
-	if c.socketPath != "" {
-		s, err := socket.ConnectToSocket(c.socketPath)
-		if err != nil {
-			log.Printf("error opening socket: %v", err)
-			c.socketConnectionErrorCount++
-		} else {
-			defer s.Close()
-		}
-		c.collectServerUpMetric(ch, s)
-		if err == nil && s != nil {
-			c.collectJailMetrics(ch, s)
-		}
-		c.collectVersionMetric(ch, s)
 	} else {
-		c.collectVersionMetric(ch, nil)
+		defer s.Close()
+	}
+
+	c.collectServerUpMetric(ch, s)
+	if err == nil && s != nil {
+		c.collectJailMetrics(ch, s)
+		c.collectVersionMetric(ch, s)
 	}
 	c.collectErrorCountMetric(ch)
 }

src/collector/f2b/database.go

@@ -1,102 +0,0 @@
-package f2b
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"log"
-)
-
-const (
-	deprecatedNamespace = "fail2ban"
-)
-
-var (
-	deprecatedMetricUp = prometheus.NewDesc(
-		prometheus.BuildFQName(deprecatedNamespace, "", "up"),
-		"(Deprecated) Was the last fail2ban query successful.",
-		nil, nil,
-	)
-	deprecatedMetricBannedIpsPerJail = prometheus.NewDesc(
-		prometheus.BuildFQName(deprecatedNamespace, "", "banned_ips"),
-		"(Deprecated) Number of banned IPs stored in the database (per jail).",
-		[]string{"jail"}, nil,
-	)
-	deprecatedMetricBadIpsPerJail = prometheus.NewDesc(
-		prometheus.BuildFQName(deprecatedNamespace, "", "bad_ips"),
-		"(Deprecated) Number of bad IPs stored in the database (per jail).",
-		[]string{"jail"}, nil,
-	)
-	deprecatedMetricEnabledJails = prometheus.NewDesc(
-		prometheus.BuildFQName(deprecatedNamespace, "", "enabled_jails"),
-		"(Deprecated) Enabled jails.",
-		[]string{"jail"}, nil,
-	)
-	deprecatedMetricErrorCount = prometheus.NewDesc(
-		prometheus.BuildFQName(deprecatedNamespace, "", "errors"),
-		"(Deprecated) Number of errors found since startup.",
-		[]string{"type"}, nil,
-	)
-)
-
-func (c *Collector) collectDeprecatedUpMetric(ch chan<- prometheus.Metric) {
-	var upMetricValue float64 = 1
-	if c.lastError != nil {
-		upMetricValue = 0
-	}
-	ch <- prometheus.MustNewConstMetric(
-		deprecatedMetricUp, prometheus.GaugeValue, upMetricValue,
-	)
-}
-
-func (c *Collector) collectDeprecatedErrorCountMetric(ch chan<- prometheus.Metric) {
-	ch <- prometheus.MustNewConstMetric(
-		deprecatedMetricErrorCount, prometheus.CounterValue, float64(c.dbErrorCount), "db",
-	)
-}
-
-func (c *Collector) collectDeprecatedBadIpsPerJailMetrics(ch chan<- prometheus.Metric) {
-	jailNameToCountMap, err := c.db.CountBadIpsPerJail()
-	c.lastError = err
-
-	if err != nil {
-		c.dbErrorCount++
-		log.Print(err)
-	}
-
-	for jailName, count := range jailNameToCountMap {
-		ch <- prometheus.MustNewConstMetric(
-			deprecatedMetricBadIpsPerJail, prometheus.GaugeValue, float64(count), jailName,
-		)
-	}
-}
-
-func (c *Collector) collectDeprecatedBannedIpsPerJailMetrics(ch chan<- prometheus.Metric) {
-	jailNameToCountMap, err := c.db.CountBannedIpsPerJail()
-	c.lastError = err
-
-	if err != nil {
-		c.dbErrorCount++
-		log.Print(err)
-	}
-
-	for jailName, count := range jailNameToCountMap {
-		ch <- prometheus.MustNewConstMetric(
-			deprecatedMetricBannedIpsPerJail, prometheus.GaugeValue, float64(count), jailName,
-		)
-	}
-}
-
-func (c *Collector) collectDeprecatedEnabledJailMetrics(ch chan<- prometheus.Metric) {
-	jailNameToEnabledMap, err := c.db.JailNameToEnabledValue()
-	c.lastError = err
-
-	if err != nil {
-		c.dbErrorCount++
-		log.Print(err)
-	}
-
-	for jailName, count := range jailNameToEnabledMap {
-		ch <- prometheus.MustNewConstMetric(
-			deprecatedMetricEnabledJails, prometheus.GaugeValue, float64(count), jailName,
-		)
-	}
-}

src/collector/f2b/socket.go

@@ -170,14 +170,10 @@ func (c *Collector) collectJailConfigMetrics(ch chan<- prometheus.Metric, s *soc
 }
 
 func (c *Collector) collectVersionMetric(ch chan<- prometheus.Metric, s *socket.Fail2BanSocket) {
-	var err error
+	fail2banVersion, err := s.GetServerVersion()
-	var fail2banVersion = ""
+	if err != nil {
-	if s != nil {
+		c.socketRequestErrorCount++
-		fail2banVersion, err = s.GetServerVersion()
+		log.Printf("failed to get fail2ban server version: %v", err)
-		if err != nil {
-			c.socketRequestErrorCount++
-			log.Printf("failed to get fail2ban server version: %v", err)
-		}
 	}
 
 	ch <- prometheus.MustNewConstMetric(

src/db/db.go

@@ -1,84 +0,0 @@
-package db
-
-import (
-	"database/sql"
-	"log"
-	"os"
-)
-
-const queryBadIpsPerJail = "SELECT j.name, (SELECT COUNT(1) FROM bips b WHERE j.name = b.jail) FROM jails j"
-const queryBannedIpsPerJail = "SELECT j.name, (SELECT COUNT(1) FROM bans b WHERE j.name = b.jail AND b.timeofban + b.bantime >= strftime('%s','now') + 0) FROM jails j"
-const queryJailNameToEnabled = "SELECT j.name, j.enabled FROM jails j"
-
-type Fail2BanDB struct {
-	DatabasePath string
-	sqliteDB     *sql.DB
-}
-
-func MustConnectToDb(databasePath string) *Fail2BanDB {
-	if _, err := os.Stat(databasePath); os.IsNotExist(err) {
-		log.Fatalf("database path does not exist: %v", err)
-	}
-	db, err := sql.Open("sqlite3", databasePath)
-	if err != nil {
-		log.Fatal(err)
-	}
-	return &Fail2BanDB{
-		DatabasePath: databasePath,
-		sqliteDB:     db,
-	}
-}
-
-func (db *Fail2BanDB) CountBannedIpsPerJail() (map[string]int, error) {
-	return db.RunJailNameToCountQuery(queryBannedIpsPerJail)
-}
-
-func (db *Fail2BanDB) CountBadIpsPerJail() (map[string]int, error) {
-	return db.RunJailNameToCountQuery(queryBadIpsPerJail)
-}
-
-func (db *Fail2BanDB) JailNameToEnabledValue() (map[string]int, error) {
-	return db.RunJailNameToCountQuery(queryJailNameToEnabled)
-}
-
-func (db *Fail2BanDB) RunJailNameToCountQuery(query string) (map[string]int, error) {
-	stmt, err := db.sqliteDB.Prepare(query)
-	defer db.mustCloseStatement(stmt)
-
-	if err != nil {
-		return nil, err
-	}
-
-	jailNameToCountMap := map[string]int{}
-	rows, err := stmt.Query()
-	if err != nil {
-		return nil, err
-	}
-	if rows == nil {
-		return jailNameToCountMap, nil
-	}
-
-	for rows.Next() {
-		if rows.Err() != nil {
-			return nil, err
-		}
-		jailName := ""
-		count := 0
-		err = rows.Scan(&jailName, &count)
-		if err != nil {
-			return nil, err
-		}
-
-		jailNameToCountMap[jailName] = count
-	}
-	return jailNameToCountMap, nil
-}
-
-func (db *Fail2BanDB) mustCloseStatement(stmt *sql.Stmt) {
-	if stmt != nil {
-		err := stmt.Close()
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-}