Merge branch 'main' into INS-172-Update-GitLab-Detector-Regex

Author: mustansir14

pkg/sources/docker/docker.go

@@ -69,9 +69,15 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	s.verify = verify
 	s.concurrency = concurrency
 
+	jobIDStr := fmt.Sprint(s.jobId)
+
 	// Reset metrics for this source at initialization time.
-	dockerImagesScanned.WithLabelValues(s.name).Set(0)
-	dockerLayersScanned.WithLabelValues(s.name).Set(0)
+	dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Set(0)
+	dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Set(0)
+	dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
+	dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
+	dockerImagesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
+	dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Set(0)
 
 	if err := anypb.UnmarshalTo(connection, &s.conn, proto.UnmarshalOptions{}); err != nil {
 		return fmt.Errorf("error unmarshalling connection: %w", err)
@@ -115,6 +121,7 @@ type layerInfo struct {
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
+	jobIDStr := fmt.Sprint(s.jobId)
 
 	workers := new(errgroup.Group)
 	workers.SetLimit(s.concurrency)
@@ -153,21 +160,23 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 			imageCtx.Logger().Error(err, "error getting image layers")
 			continue
 		}
+		dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(layers)))
 
 		// Get history entries and associate them with layers
 		historyEntries, err := getHistoryEntries(imageCtx, imgInfo, layers)
 		if err != nil {
 			imageCtx.Logger().Error(err, "error getting image history entries")
 			continue
 		}
+		dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(historyEntries)))
 
 		// Scan each history entry for secrets in build commands
 		for _, historyEntry := range historyEntries {
 			if err := s.processHistoryEntry(imageCtx, historyEntry, chunksChan); err != nil {
 				imageCtx.Logger().Error(err, "error processing history entry")
 				continue
 			}
-			dockerHistoryEntriesScanned.WithLabelValues(s.name).Inc()
+			dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Inc()
 		}
 
 		imageCtx.Logger().V(2).Info("scanning image layers")
@@ -179,7 +188,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 					imageCtx.Logger().Error(err, "error processing layer")
 					return nil
 				}
-				dockerLayersScanned.WithLabelValues(s.name).Inc()
+				dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Inc()
 
 				return nil
 			})
@@ -190,7 +199,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 			continue
 		}
 
-		dockerImagesScanned.WithLabelValues(s.name).Inc()
+		dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Inc()
 	}
 
 	return nil
@@ -502,7 +511,7 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 	}
 
 	var opts []remote.Option
-	opts = append(opts, remote.WithTransport(defaultTransport))
+	opts = append(opts, remote.WithTransport(common.NewInstrumentedTransport(common.NewCustomTransport(defaultTransport))))
 
 	// Configure authentication based on credential type
 	switch s.conn.GetCredential().(type) {

pkg/sources/docker/docker_test.go

@@ -109,9 +109,9 @@ func TestQuayRegistry(t *testing.T) {
 	close(chunksChan)
 	wg.Wait()
 
-	assert.Equal(t, 945, chunkCounter)
+	assert.Equal(t, 944, chunkCounter)
 	assert.Equal(t, 941, layerCounter)
-	assert.Equal(t, 4, historyCounter)
+	assert.Equal(t, 3, historyCounter)
 }
 
 func TestGHCRRegistry(t *testing.T) {

pkg/sources/docker/metrics.go

@@ -14,23 +14,48 @@ var (
 		Name:      "docker_layers_scanned",
 		Help:      "Total number of Docker layers scanned.",
 	},
-		[]string{"source_name"})
+		[]string{"source_name", "job_id"})
+
+	dockerLayersEnumerated = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: common.MetricsNamespace,
+			Subsystem: common.MetricsSubsystem,
+			Name:      "docker_layers_enumerated",
+			Help:      "Total number of Docker layers enumerated.",
+		},
+		[]string{"source_name", "job_id"})
 
 	dockerHistoryEntriesScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: common.MetricsNamespace,
 		Subsystem: common.MetricsSubsystem,
 		Name:      "docker_history_entries_scanned",
 		Help:      "Total number of Docker image history entries scanned.",
 	},
-		[]string{"source_name"})
+		[]string{"source_name", "job_id"})
+
+	dockerHistoryEntriesEnumerated = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: common.MetricsNamespace,
+		Subsystem: common.MetricsSubsystem,
+		Name:      "docker_history_entries_enumerated",
+		Help:      "Total number of Docker history entries enumerated.",
+	},
+		[]string{"source_name", "job_id"})
 
 	dockerImagesScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: common.MetricsNamespace,
 		Subsystem: common.MetricsSubsystem,
 		Name:      "docker_images_scanned",
 		Help:      "Total number of Docker images scanned.",
 	},
-		[]string{"source_name"})
+		[]string{"source_name", "job_id"})
+
+	dockerImagesEnumerated = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: common.MetricsNamespace,
+		Subsystem: common.MetricsSubsystem,
+		Name:      "docker_images_enumerated",
+		Help:      "Total number of Docker images enumerated.",
+	},
+		[]string{"source_name", "job_id"})
 
 	dockerListImagesAPIDuration = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{