refactor: Sync reporter to allow env resolver adapter (#495)

* refactor: Sync reporter to allow env resolver adapter

* fix: Set optional params only when not empty

* fix: linter warning
This commit is contained in:
Abhisek Datta 2025-05-27 22:00:50 +05:30 committed by GitHub
parent 1f8a5750d2
commit 72e08bdd8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 226 additions and 53 deletions

4
go.mod
View File

@ -3,8 +3,8 @@ module github.com/safedep/vet
go 1.24.2 go 1.24.2
require ( require (
buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250418165058-162f6b0cc319.2 buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250527100058-ba6815156f1f.2
buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250418165058-162f6b0cc319.1 buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250527100058-ba6815156f1f.1
entgo.io/ent v0.14.4 entgo.io/ent v0.14.4
github.com/AlecAivazis/survey/v2 v2.3.7 github.com/AlecAivazis/survey/v2 v2.3.7
github.com/BurntSushi/toml v1.5.0 github.com/BurntSushi/toml v1.5.0

4
go.sum
View File

@ -8,8 +8,12 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-2025030720450
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250307204501-0409229c3780.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250307204501-0409229c3780.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U=
buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250418165058-162f6b0cc319.2 h1:txDywYkqsXvtA/sDSMcwMjC8XTHnqlyc+3aIFMzRVeQ= buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250418165058-162f6b0cc319.2 h1:txDywYkqsXvtA/sDSMcwMjC8XTHnqlyc+3aIFMzRVeQ=
buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250418165058-162f6b0cc319.2/go.mod h1:fdc8Y27iNGRno4W5CMBFBOQlG75NJ3M9q0GZRvY/TVA= buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250418165058-162f6b0cc319.2/go.mod h1:fdc8Y27iNGRno4W5CMBFBOQlG75NJ3M9q0GZRvY/TVA=
buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250527100058-ba6815156f1f.2 h1:WfzhSnTtAbi5Zn9GwgNkhPDTsQD8ismpUukHrxvPGLQ=
buf.build/gen/go/safedep/api/grpc/go v1.5.1-20250527100058-ba6815156f1f.2/go.mod h1:HNLXNUIlZB8rXnvWj0hVVze1ioWtUa+GauXoM85qpkI=
buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250418165058-162f6b0cc319.1 h1:vJeI1IQuxGZd9r5RHNiLTJ+aPQfy0g7h3Gqa9/Ql7kA= buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250418165058-162f6b0cc319.1 h1:vJeI1IQuxGZd9r5RHNiLTJ+aPQfy0g7h3Gqa9/Ql7kA=
buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250418165058-162f6b0cc319.1/go.mod h1:uR95GqsnNCRn6cTyRBte6uMJMm0rEBRxTGpakKCNL9I= buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250418165058-162f6b0cc319.1/go.mod h1:uR95GqsnNCRn6cTyRBte6uMJMm0rEBRxTGpakKCNL9I=
buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250527100058-ba6815156f1f.1 h1:Fv2LmHer2zT/JBuxQEi336gMmG62lbRnCcNuHhg0MvA=
buf.build/gen/go/safedep/api/protocolbuffers/go v1.36.6-20250527100058-ba6815156f1f.1/go.mod h1:uR95GqsnNCRn6cTyRBte6uMJMm0rEBRxTGpakKCNL9I=
cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg=
cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=

View File

@ -7,6 +7,7 @@ import (
"sync" "sync"
"buf.build/gen/go/safedep/api/grpc/go/safedep/services/controltower/v1/controltowerv1grpc" "buf.build/gen/go/safedep/api/grpc/go/safedep/services/controltower/v1/controltowerv1grpc"
controltowerv1pb "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/controltower/v1"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
policyv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/policy/v1" policyv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/policy/v1"
vulnerabilityv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/vulnerability/v1" vulnerabilityv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/vulnerability/v1"
@ -23,6 +24,55 @@ import (
const syncReporterDefaultWorkerCount = 10 const syncReporterDefaultWorkerCount = 10
// Contract for implementing environment resolver for the sync reporter.
// Here we decouple the the actual implementation of the resolver to the
// client that uses the reporter plugin. The resolver is used to provide
// environment awareness to the reporter. For example, when running in GitHub
// or on a Git repository, the resolver can provide the project source, project
// and other information that is required to create a tool session.
type SyncReporterEnvResolver interface {
// The resolved source of the runtime environment (e.g. GitHub)
GetProjectSource() controltowerv1pb.Project_Source
// The resolved URL of the runtime environment (e.g. GitHub repository URL)
GetProjectUrl() string
// The trigger of the runtime environment (e.g. CI/CD pipeline)
Trigger() controltowerv1.ToolTrigger
// The Git reference of the runtime environment (e.g. branch, tag, commit)
GitRef() string
// The Git SHA of the runtime environment (e.g. commit hash)
GitSha() string
}
type defaultSyncReporterEnvResolver struct{}
func (r *defaultSyncReporterEnvResolver) GetProjectSource() controltowerv1pb.Project_Source {
return controltowerv1pb.Project_SOURCE_UNSPECIFIED
}
func (r *defaultSyncReporterEnvResolver) GetProjectUrl() string {
return ""
}
func (r *defaultSyncReporterEnvResolver) GitRef() string {
return ""
}
func (r *defaultSyncReporterEnvResolver) GitSha() string {
return ""
}
func (r *defaultSyncReporterEnvResolver) Trigger() controltowerv1.ToolTrigger {
return controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL
}
func DefaultSyncReporterEnvResolver() SyncReporterEnvResolver {
return &defaultSyncReporterEnvResolver{}
}
type SyncReporterConfig struct { type SyncReporterConfig struct {
// gRPC connection for ControlTower // gRPC connection for ControlTower
ClientConnection *grpc.ClientConn ClientConnection *grpc.ClientConn
@ -31,16 +81,9 @@ type SyncReporterConfig struct {
// In this case, a new project is created per package manifest // In this case, a new project is created per package manifest
EnableMultiProjectSync bool EnableMultiProjectSync bool
// Required // Required when scanning a single project
ProjectName string ProjectName string
ProjectVersion string ProjectVersion string
TriggerEvent string
// Optional or auto-discovered from environment
GitRef string
GitRefName string
GitRefType string
GitSha string
// Performance // Performance
WorkerCount int WorkerCount int
@ -123,32 +166,42 @@ type workItem struct {
} }
type syncReporter struct { type syncReporter struct {
config *SyncReporterConfig config *SyncReporterConfig
workQueue chan *workItem workQueue chan *workItem
done chan bool done chan bool
wg sync.WaitGroup wg sync.WaitGroup
client *grpc.ClientConn client *grpc.ClientConn
sessions *syncSessionPool sessions *syncSessionPool
callbacks SyncReporterCallbacks envResolver SyncReporterEnvResolver
callbacks SyncReporterCallbacks
} }
// Verify syncReporter implements the Reporter interface // Verify syncReporter implements the Reporter interface
var _ Reporter = (*syncReporter)(nil) var _ Reporter = (*syncReporter)(nil)
func NewSyncReporter(config SyncReporterConfig, callbacks SyncReporterCallbacks) (Reporter, error) { func NewSyncReporter(config SyncReporterConfig, envResolver SyncReporterEnvResolver, callbacks SyncReporterCallbacks) (*syncReporter, error) {
if config.ClientConnection == nil { if config.ClientConnection == nil {
return nil, fmt.Errorf("missing gRPC client connection") return nil, fmt.Errorf("missing gRPC client connection")
} }
// TODO: Auto-discover config using CI environment variables if envResolver == nil {
// if enabled by the user return nil, fmt.Errorf("missing environment resolver")
}
syncSessionPool := syncSessionPool{ syncSessionPool := syncSessionPool{
syncSessions: make(map[string]syncSession), syncSessions: make(map[string]syncSession),
} }
trigger := controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL done := make(chan bool)
source := packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED self := &syncReporter{
config: &config,
done: done,
workQueue: make(chan *workItem, 1000),
client: config.ClientConnection,
sessions: &syncSessionPool,
callbacks: callbacks,
envResolver: envResolver,
}
// A multi-project sync is required for cases like GitHub org where // A multi-project sync is required for cases like GitHub org where
// we are scanning multiple repositories // we are scanning multiple repositories
@ -159,14 +212,7 @@ func NewSyncReporter(config SyncReporterConfig, callbacks SyncReporterCallbacks)
// Refactor this into a common session creator function // Refactor this into a common session creator function
toolServiceClient := controltowerv1grpc.NewToolServiceClient(config.ClientConnection) toolServiceClient := controltowerv1grpc.NewToolServiceClient(config.ClientConnection)
toolSessionRes, err := toolServiceClient.CreateToolSession(context.Background(), toolSessionRes, err := toolServiceClient.CreateToolSession(context.Background(),
&controltowerv1.CreateToolSessionRequest{ self.createToolSessionRequestForProjectVersion(config.ProjectName, config.ProjectVersion))
ToolName: config.Tool.Name,
ToolVersion: config.Tool.Version,
ProjectName: config.ProjectName,
ProjectVersion: &config.ProjectVersion,
ProjectSource: &source,
Trigger: &trigger,
})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create tool session: %w", err) return nil, fmt.Errorf("failed to create tool session: %w", err)
} }
@ -178,16 +224,6 @@ func NewSyncReporter(config SyncReporterConfig, callbacks SyncReporterCallbacks)
toolServiceClient) toolServiceClient)
} }
done := make(chan bool)
self := &syncReporter{
config: &config,
done: done,
workQueue: make(chan *workItem, 1000),
client: config.ClientConnection,
sessions: &syncSessionPool,
callbacks: callbacks,
}
self.dispatchOnSyncStart() self.dispatchOnSyncStart()
self.startWorkers() self.startWorkers()
return self, nil return self, nil
@ -203,23 +239,12 @@ func (s *syncReporter) AddManifest(manifest *models.PackageManifest) {
projectName := manifest.GetSource().GetNamespace() projectName := manifest.GetSource().GetNamespace()
projectVersion := "main" projectVersion := "main"
source := packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED
trigger := controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL
logger.Debugf("Report Sync: Creating tool session for project: %s, version: %s", logger.Debugf("Report Sync: Creating tool session for project: %s, version: %s",
projectName, projectVersion) projectName, projectVersion)
// Refactor this into a common session creator function
toolServiceClient := controltowerv1grpc.NewToolServiceClient(s.client) toolServiceClient := controltowerv1grpc.NewToolServiceClient(s.client)
toolSessionRes, err := toolServiceClient.CreateToolSession(context.Background(), toolSessionRes, err := toolServiceClient.CreateToolSession(context.Background(),
&controltowerv1.CreateToolSessionRequest{ s.createToolSessionRequestForProjectVersion(projectName, projectVersion))
ToolName: s.config.Tool.Name,
ToolVersion: s.config.Tool.Version,
ProjectName: projectName,
ProjectVersion: &projectVersion,
ProjectSource: &source,
Trigger: &trigger,
})
if err != nil { if err != nil {
logger.Errorf("failed to create tool session for project: %s/%s: %v", logger.Errorf("failed to create tool session for project: %s/%s: %v",
projectName, projectVersion, err) projectName, projectVersion, err)
@ -547,3 +572,42 @@ func (s *syncReporter) syncPackage(pkg *models.Package) error {
s.dispatchOnPackageSyncDone(pkg) s.dispatchOnPackageSyncDone(pkg)
return nil return nil
} }
func (s *syncReporter) createToolSessionRequestForProjectVersion(projectName, projectVersion string) *controltowerv1.CreateToolSessionRequest {
source := packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED
trigger := s.envResolver.Trigger()
originSource := s.envResolver.GetProjectSource()
originUrl := s.envResolver.GetProjectUrl()
gitRef := s.envResolver.GitRef()
gitSha := s.envResolver.GitSha()
req := &controltowerv1.CreateToolSessionRequest{
ToolName: s.config.Tool.Name,
ToolVersion: s.config.Tool.Version,
ProjectName: projectName,
ProjectVersion: &projectVersion,
ProjectSource: &source,
}
if trigger != controltowerv1.ToolTrigger_TOOL_TRIGGER_UNSPECIFIED {
req.Trigger = &trigger
}
if originSource != controltowerv1pb.Project_SOURCE_UNSPECIFIED {
req.OriginProjectSource = &originSource
}
if originUrl != "" {
req.OriginProjectUrl = &originUrl
}
if gitRef != "" {
req.GitRef = &gitRef
}
if gitSha != "" {
req.GitSha = &gitSha
}
return req
}

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"testing" "testing"
controltowerv1pb "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/controltower/v1"
malysisv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/malysis/v1" malysisv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/malysis/v1"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
vulnerabilityv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/vulnerability/v1" vulnerabilityv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/vulnerability/v1"
@ -571,3 +572,107 @@ func TestSyncEvent(t *testing.T) {
}) })
} }
} }
type testEnvResolver struct {
projectSource controltowerv1pb.Project_Source
projectUrl string
trigger controltowerv1.ToolTrigger
gitRef string
gitSha string
}
func (r *testEnvResolver) GetProjectSource() controltowerv1pb.Project_Source {
return r.projectSource
}
func (r *testEnvResolver) GetProjectUrl() string {
return r.projectUrl
}
func (r *testEnvResolver) Trigger() controltowerv1.ToolTrigger {
return r.trigger
}
func (r *testEnvResolver) GitRef() string {
return r.gitRef
}
func (r *testEnvResolver) GitSha() string {
return r.gitSha
}
func TestCreateToolSessionRequestForProjectVersion(t *testing.T) {
cases := []struct {
name string
envResolver SyncReporterEnvResolver
config *SyncReporterConfig
assertFn func(t *testing.T, request *controltowerv1.CreateToolSessionRequest)
}{
{
name: "default",
config: &SyncReporterConfig{
Tool: ToolMetadata{
Name: "test-tool",
Version: "1.0.0",
},
ProjectName: "test-project",
ProjectVersion: "1.0.0",
},
envResolver: DefaultSyncReporterEnvResolver(),
assertFn: func(t *testing.T, request *controltowerv1.CreateToolSessionRequest) {
assert.Equal(t, "test-tool", request.ToolName)
assert.Equal(t, "1.0.0", request.ToolVersion)
assert.Equal(t, "test-project", request.ProjectName)
assert.Equal(t, "1.0.0", *request.ProjectVersion)
assert.Equal(t, packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED, *request.ProjectSource)
assert.Equal(t, controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL, *request.Trigger)
assert.Nil(t, request.OriginProjectSource)
assert.Nil(t, request.OriginProjectUrl)
assert.Nil(t, request.GitRef)
assert.Nil(t, request.GitSha)
},
},
{
name: "with resolved attributes",
config: &SyncReporterConfig{
Tool: ToolMetadata{
Name: "test-tool",
Version: "1.0.0",
},
ProjectName: "test-project",
ProjectVersion: "1.0.0",
},
envResolver: &testEnvResolver{
projectSource: controltowerv1pb.Project_SOURCE_GITHUB,
projectUrl: "https://github.com/test/test",
trigger: controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL,
gitRef: "refs/heads/main",
gitSha: "1234567890",
},
assertFn: func(t *testing.T, request *controltowerv1.CreateToolSessionRequest) {
assert.Equal(t, "test-tool", request.ToolName)
assert.Equal(t, "1.0.0", request.ToolVersion)
assert.Equal(t, "test-project", request.ProjectName)
assert.Equal(t, "1.0.0", *request.ProjectVersion)
assert.Equal(t, packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED, *request.ProjectSource)
assert.Equal(t, controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL, *request.Trigger)
assert.Equal(t, controltowerv1pb.Project_SOURCE_GITHUB, *request.OriginProjectSource)
assert.Equal(t, "https://github.com/test/test", *request.OriginProjectUrl)
assert.Equal(t, "refs/heads/main", *request.GitRef)
assert.Equal(t, "1234567890", *request.GitSha)
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
sr := &syncReporter{
config: tc.config,
envResolver: tc.envResolver,
}
request := sr.createToolSessionRequestForProjectVersion(tc.config.ProjectName, tc.config.ProjectVersion)
tc.assertFn(t, request)
})
}
}

View File

@ -647,7 +647,7 @@ func internalStartScan() error {
ProjectVersion: syncReportStream, ProjectVersion: syncReportStream,
EnableMultiProjectSync: syncEnableMultiProject, EnableMultiProjectSync: syncEnableMultiProject,
ClientConnection: clientConn, ClientConnection: clientConn,
}, reporter.SyncReporterCallbacks{ }, reporter.DefaultSyncReporterEnvResolver(), reporter.SyncReporterCallbacks{
OnSyncStart: func() { OnSyncStart: func() {
ui.PrintMsg("🌐 Syncing data to SafeDep Cloud...") ui.PrintMsg("🌐 Syncing data to SafeDep Cloud...")
}, },