Switch to label-based gauges

fuzzer.go

@@ -2,73 +2,25 @@ package main
 
 import (
 	"fmt"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"io/ioutil"
-	"log"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 )
 
 // Represents a Fuzzer and its state, as reported in the "fuzzer_stats" file
 type Fuzzer struct {
-	fuzzer_pid        prometheus.Gauge
+	afl_banner    string
-	cycles_done       prometheus.Gauge
+	afl_directory string
-	execs_done        prometheus.Gauge
-	execs_per_sec     prometheus.Gauge
-	paths_total       prometheus.Gauge
-	paths_favored     prometheus.Gauge
-	paths_found       prometheus.Gauge
-	paths_imported    prometheus.Gauge
-	max_depth         prometheus.Gauge
-	cur_path          prometheus.Gauge
-	pending_favs      prometheus.Gauge
-	pending_total     prometheus.Gauge
-	variable_paths    prometheus.Gauge
-	stability         prometheus.Gauge
-	bitmap_cvg        prometheus.Gauge
-	unique_crashes    prometheus.Gauge
-	unique_hangs      prometheus.Gauge
-	last_path         prometheus.Gauge
-	last_crash        prometheus.Gauge
-	last_hang         prometheus.Gauge
-	execs_since_crash prometheus.Gauge
-	exec_timeout      prometheus.Gauge
-	slowest_exec_ms   prometheus.Gauge
-	peak_rss_mb       prometheus.Gauge
-	afl_banner        string
-	afl_directory     string
-	// missing start_time, last_update, afl_version, target_mode and command_line, and to be honest, I don't see a reason to export these values via prometheus
 }
 
-// Initialises all gauges. Needs afl_banner to be set properly before (and doesn't check that on its own)
+// Creates a fuzzer from the given target path
-func (f *Fuzzer) initGauges() {
+func CreateFuzzer(target string) Fuzzer {
-	log.Printf("%s %s", f.afl_directory, f.afl_banner)
+	return Fuzzer{
-	f.fuzzer_pid = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_fuzzer_pid"})
+		afl_banner:    path.Base(target),
-	f.cycles_done = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_cycles_done"})
+		afl_directory: target,
-	f.execs_done = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_execs_done"})
+	}
-	f.execs_per_sec = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_execs_per_sec"})
-	f.paths_total = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_paths_total"})
-	f.paths_favored = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_paths_favored"})
-	f.paths_found = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_paths_found"})
-	f.paths_imported = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_paths_imported"})
-	f.max_depth = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_max_depth"})
-	f.cur_path = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_cur_path"})
-	f.pending_favs = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_pending_favs"})
-	f.pending_total = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_pending_total"})
-	f.variable_paths = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_variable_paths"})
-	f.stability = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_stability"})
-	f.bitmap_cvg = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_bitmap_cvg"})
-	f.unique_crashes = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_unique_crashes"})
-	f.unique_hangs = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_unique_hangs"})
-	f.last_path = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_last_path"})
-	f.last_crash = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_last_crash"})
-	f.last_hang = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_last_hang"})
-	f.execs_since_crash = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_execs_since_crash"})
-	f.exec_timeout = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_exec_timeout"})
-	f.slowest_exec_ms = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_slowest_exec_ms"})
-	f.peak_rss_mb = promauto.NewGauge(prometheus.GaugeOpts{Name: f.afl_banner + "_peak_rss_mb"})
 }
 
 // Parses the "fuzzer_stats" file present in the given directory, and updates the gauges of the Fuzzer instance accordingly
@@ -83,115 +35,92 @@ func (f *Fuzzer) ParseStatsFile() error {
 	// Split file contents into lines
 	fileLines := strings.Split(string(fileBytes), "\n")
 
-	// So we want to walk over the file two times.
+	for _, l := range fileLines {
-	//  1) If it is unset: read "afl_banner", set it, and run initGauges()
+		// Skip unparseable files
-	//  2) Set every other value to its corresponding gauge.
+		if !strings.Contains(l, ":") {
-	bannerSet := f.afl_banner != "" // 1)
+			continue
-	gaugesSet := false // 2)
+		}
-	for !(bannerSet && gaugesSet) {
-		// Iterate over every line
-		for _, l := range fileLines {
-			// Skip empty lines
-			if l == "" {
-				continue
-			}
 
-			// Convert line to key and value
+		// Convert line to key and value
-			parts := strings.SplitN(l, ":", 2)
+		parts := strings.SplitN(l, ":", 2)
-			key := strings.TrimSpace(parts[0])
+		key := strings.TrimSpace(parts[0])
-			val := strings.TrimSpace(parts[1])
+		val := strings.TrimSpace(parts[1])
 
-			// 1)
+		switch key {
-			if !bannerSet {
+		case "fuzzer_pid":
-				if key == "afl_banner" {
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-					f.afl_banner = val
+			fuzzer_pid.WithLabelValues(f.afl_banner).Set(float64(convVal))
-					bannerSet = true
+		case "cycles_done":
-					f.initGauges()
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-					break
+			cycles_done.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				} else {
+		case "execs_done":
-					continue
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				}
+			execs_done.WithLabelValues(f.afl_banner).Set(float64(convVal))
-			}
+		case "execs_per_sec":
-			
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-			// 2)
+			execs_per_sec.WithLabelValues(f.afl_banner).Set(float64(convVal))
-			switch key {
+		case "paths_total":
-			case "fuzzer_pid":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			paths_total.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.fuzzer_pid.Set(float64(convVal))
+		case "paths_favored":
-			case "cycles_done":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			paths_favored.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.cycles_done.Set(float64(convVal))
+		case "paths_found":
-			case "execs_done":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			paths_found.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.execs_done.Set(float64(convVal))
+		case "paths_imported":
-			case "execs_per_sec":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			paths_imported.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.execs_per_sec.Set(float64(convVal))
+		case "max_depth":
-			case "paths_total":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			max_depth.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.paths_total.Set(float64(convVal))
+		case "cur_path":
-			case "paths_favored":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			cur_path.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.paths_favored.Set(float64(convVal))
+		case "pending_favs":
-			case "paths_found":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			pending_favs.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.paths_found.Set(float64(convVal))
+		case "pending_total":
-			case "paths_imported":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			pending_total.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.paths_imported.Set(float64(convVal))
+		case "variable_paths":
-			case "max_depth":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			variable_paths.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.max_depth.Set(float64(convVal))
+		case "stability":
-			case "cur_path":
+			val = strings.Replace(val, "%", "", 1)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			convVal, _ := strconv.ParseFloat(val, 64)
-				f.cur_path.Set(float64(convVal))
+			stability.WithLabelValues(f.afl_banner).Set(convVal)
-			case "pending_favs":
+		case "bitmap_cvg":
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			val = strings.Replace(val, "%", "", 1)
-				f.pending_favs.Set(float64(convVal))
+			convVal, _ := strconv.ParseFloat(val, 64)
-			case "pending_total":
+			bitmap_cvg.WithLabelValues(f.afl_banner).Set(convVal)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+		case "unique_crashes":
-				f.pending_total.Set(float64(convVal))
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-			case "variable_paths":
+			unique_crashes.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+		case "unique_hangs":
-				f.variable_paths.Set(float64(convVal))
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-			case "stability":
+			unique_hangs.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				val = strings.Replace(val, "%", "", 1)
+		case "last_path":
-				convVal, _ := strconv.ParseFloat(val, 64)
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				f.stability.Set(convVal)
+			last_path.WithLabelValues(f.afl_banner).Set(float64(convVal))
-			case "bitmap_cvg":
+		case "last_crash":
-				val = strings.Replace(val, "%", "", 1)
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseFloat(val, 64)
+			last_crash.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.bitmap_cvg.Set(convVal)
+		case "last_hang":
-			case "unique_crashes":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			last_hang.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.unique_crashes.Set(float64(convVal))
+		case "execs_since_crash":
-			case "unique_hangs":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			execs_since_crash.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.unique_hangs.Set(float64(convVal))
+		case "exec_timeout":
-			case "last_path":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			exec_timeout.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.last_path.Set(float64(convVal))
+		case "slowest_exec_ms":
-			case "last_crash":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			slowest_exec_ms.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.last_crash.Set(float64(convVal))
+		case "peak_rss_mb":
-			case "last_hang":
+			convVal, _ := strconv.ParseInt(val, 10, 64)
-				convVal, _ := strconv.ParseInt(val, 10, 64)
+			peak_rss_mb.WithLabelValues(f.afl_banner).Set(float64(convVal))
-				f.last_hang.Set(float64(convVal))
-			case "execs_since_crash":
-				convVal, _ := strconv.ParseInt(val, 10, 64)
-				f.execs_since_crash.Set(float64(convVal))
-			case "exec_timeout":
-				convVal, _ := strconv.ParseInt(val, 10, 64)
-				f.exec_timeout.Set(float64(convVal))
-			case "slowest_exec_ms":
-				convVal, _ := strconv.ParseInt(val, 10, 64)
-				f.slowest_exec_ms.Set(float64(convVal))
-			case "peak_rss_mb":
-				convVal, _ := strconv.ParseInt(val, 10, 64)
-				f.peak_rss_mb.Set(float64(convVal))
-			}
-
-			gaugesSet = true
 		}
 	}
 

gauges.go

@@ -0,0 +1,86 @@
+package main
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	fuzzer_pid        *prometheus.GaugeVec
+	cycles_done       *prometheus.GaugeVec
+	execs_done        *prometheus.GaugeVec
+	execs_per_sec     *prometheus.GaugeVec
+	paths_total       *prometheus.GaugeVec
+	paths_favored     *prometheus.GaugeVec
+	paths_found       *prometheus.GaugeVec
+	paths_imported    *prometheus.GaugeVec
+	max_depth         *prometheus.GaugeVec
+	cur_path          *prometheus.GaugeVec
+	pending_favs      *prometheus.GaugeVec
+	pending_total     *prometheus.GaugeVec
+	variable_paths    *prometheus.GaugeVec
+	stability         *prometheus.GaugeVec
+	bitmap_cvg        *prometheus.GaugeVec
+	unique_crashes    *prometheus.GaugeVec
+	unique_hangs      *prometheus.GaugeVec
+	last_path         *prometheus.GaugeVec
+	last_crash        *prometheus.GaugeVec
+	last_hang         *prometheus.GaugeVec
+	execs_since_crash *prometheus.GaugeVec
+	exec_timeout      *prometheus.GaugeVec
+	slowest_exec_ms   *prometheus.GaugeVec
+	peak_rss_mb       *prometheus.GaugeVec
+	// missing start_time, last_update, afl_version, target_mode and command_line, and to be honest, I don't see a reason to export these values via prometheus
+)
+
+// Initializes all gauges and register them
+func InitializeGauges() {
+	// Set up gauges
+	fuzzer_pid = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "fuzzer_pid"}, []string{"name"})
+	cycles_done = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "cycles_done"}, []string{"name"})
+	execs_done = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "execs_done"}, []string{"name"})
+	execs_per_sec = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "execs_per_sec"}, []string{"name"})
+	paths_total = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "paths_total"}, []string{"name"})
+	paths_favored = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "paths_favored"}, []string{"name"})
+	paths_found = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "paths_found"}, []string{"name"})
+	paths_imported = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "paths_imported"}, []string{"name"})
+	max_depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "max_depth"}, []string{"name"})
+	cur_path = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "cur_path"}, []string{"name"})
+	pending_favs = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "pending_favs"}, []string{"name"})
+	pending_total = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "pending_total"}, []string{"name"})
+	variable_paths = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "variable_paths"}, []string{"name"})
+	stability = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "stability"}, []string{"name"})
+	bitmap_cvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "bitmap_cvg"}, []string{"name"})
+	unique_crashes = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "unique_crashes"}, []string{"name"})
+	unique_hangs = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "unique_hangs"}, []string{"name"})
+	last_path = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "last_path"}, []string{"name"})
+	last_crash = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "last_crash"}, []string{"name"})
+	last_hang = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "last_hang"}, []string{"name"})
+	execs_since_crash = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "execs_since_crash"}, []string{"name"})
+	exec_timeout = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "exec_timeout"}, []string{"name"})
+	slowest_exec_ms = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "slowest_exec_ms"}, []string{"name"})
+	peak_rss_mb = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "peak_rss_mb"}, []string{"name"})
+
+	// Register gauges
+	prometheus.MustRegister(fuzzer_pid)
+	prometheus.MustRegister(cycles_done)
+	prometheus.MustRegister(execs_done)
+	prometheus.MustRegister(execs_per_sec)
+	prometheus.MustRegister(paths_total)
+	prometheus.MustRegister(paths_favored)
+	prometheus.MustRegister(paths_found)
+	prometheus.MustRegister(paths_imported)
+	prometheus.MustRegister(max_depth)
+	prometheus.MustRegister(cur_path)
+	prometheus.MustRegister(pending_favs)
+	prometheus.MustRegister(pending_total)
+	prometheus.MustRegister(variable_paths)
+	prometheus.MustRegister(stability)
+	prometheus.MustRegister(bitmap_cvg)
+	prometheus.MustRegister(unique_crashes)
+	prometheus.MustRegister(unique_hangs)
+	prometheus.MustRegister(last_path)
+	prometheus.MustRegister(last_crash)
+	prometheus.MustRegister(last_hang)
+	prometheus.MustRegister(execs_since_crash)
+	prometheus.MustRegister(exec_timeout)
+	prometheus.MustRegister(slowest_exec_ms)
+	prometheus.MustRegister(peak_rss_mb)
+}

watch.go

@@ -26,19 +26,14 @@ func getFuzzersToWatch() ([]string, error) {
 
 // Registers the given paths as fuzzer directories which should be monitored
 func registerFuzzers(targets []string) {
+	// First, create fuzzer instances based on the directory
 	for _, f := range targets {
-		var tmpFuzzer Fuzzer
+		tmpFuzzer := CreateFuzzer(f)
-		tmpFuzzer.afl_directory = f
-		parseErr := tmpFuzzer.ParseStatsFile()
-
-		if parseErr != nil {
-			log.Printf("Encountered error while parsing %s: %s", f, parseErr.Error())
-			break
-		}
-
-		// Append fuzzer to our list of registered fuzzers
 		registeredFuzzers = append(registeredFuzzers, tmpFuzzer)
 	}
+
+	// Create gauges
+	InitializeGauges()
 }
 
 // Watch over the fuzzer(s)