diff --git a/cmd/cloud/query.go b/cmd/cloud/query.go index f5fac05..f9f8f22 100644 --- a/cmd/cloud/query.go +++ b/cmd/cloud/query.go @@ -2,11 +2,13 @@ package cloud import ( "errors" + "fmt" "os" "sort" "github.com/jedib0t/go-pretty/v6/table" "github.com/safedep/vet/internal/auth" + "github.com/safedep/vet/internal/ui" "github.com/safedep/vet/pkg/cloud/query" "github.com/safedep/vet/pkg/common/logger" "github.com/spf13/cobra" @@ -82,6 +84,8 @@ func renderQueryResponseAsTable(response *query.QueryResponse) error { return nil } + ui.PrintSuccess(fmt.Sprintf("Query returned %d results", response.Count())) + // Header headers := []string{} response.GetRow(0).ForEachField(func(key string, _ interface{}) { diff --git a/go.mod b/go.mod index 8afee65..9ac7fea 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/safedep/vet go 1.22.1 require ( - buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241009065537-4ffd9786549f.1 - buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241009065537-4ffd9786549f.1 + buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241011110723-95b33664baad.1 + buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241011110723-95b33664baad.1 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/CycloneDX/cyclonedx-go v0.9.0 github.com/anchore/syft v1.11.1 diff --git a/go.sum b/go.sum index afe04c0..56b3b61 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,17 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240508200655-46a4cf4ba109.1 h1:PvjTFY+MqrYYulH74R8ddQodrEP1sThjkba8kcqO2dM= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240508200655-46a4cf4ba109.1/go.mod h1:Duw/9JoXkXIydyASnLYIiufkzySThoqavOsF+IihqvM= +buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241008111718-480f2b42b425.1 h1:ofDpCRg/yPfD0UMny+yaq4G5sHLNFhBH2DtIN9orEzQ= +buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241008111718-480f2b42b425.1/go.mod h1:DGREO91B1aVG1+zfN4gT/ZQZ0Ykcv+TZGf18SeGDcMc= buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241009065537-4ffd9786549f.1 h1:7+y3t/2/6sEf63xCX9ErNdsuFzB0Q18PIT14aAxz5wo= buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241009065537-4ffd9786549f.1/go.mod h1:Io32CuF5oJZDSkWJVgKvfOBTqSHnC7mqh9dpqq/PSNA= +buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241011110723-95b33664baad.1 h1:bUOILp76S06U58bp3EWI6qsoeEEBGGQJeMFdQ5Otc34= +buf.build/gen/go/safedep/api/grpc/go v1.5.1-20241011110723-95b33664baad.1/go.mod h1:LSghi5M4Z+uYLyo85Bs6lVpzit6ZoSph8YOyvpHL3mY= +buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241008111718-480f2b42b425.1 h1:IziBLru8MusSLO2lNDzSxbVxKu0dGI8zE8vG+tCMYjI= +buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241008111718-480f2b42b425.1/go.mod h1:WCxZaBpYxgWnSpauuzVhzbJawAp6uPXJPN5tbDpceQ0= buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241009065537-4ffd9786549f.1 h1:F4fk9nhlB2ZU8odF5S/w/nS8r0htIbu6eKDKDBP+kzQ= buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241009065537-4ffd9786549f.1/go.mod h1:WCxZaBpYxgWnSpauuzVhzbJawAp6uPXJPN5tbDpceQ0= +buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241011110723-95b33664baad.1 h1:HMED4kNSQJgrmFZXorIP1mpVbnKxzvhMgc3BOawTtGw= +buf.build/gen/go/safedep/api/protocolbuffers/go v1.35.1-20241011110723-95b33664baad.1/go.mod h1:WCxZaBpYxgWnSpauuzVhzbJawAp6uPXJPN5tbDpceQ0= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= diff --git a/internal/auth/grpc.go b/internal/auth/grpc.go index fe096d0..e2de9df 100644 --- a/internal/auth/grpc.go +++ b/internal/auth/grpc.go @@ -33,12 +33,13 @@ func cloudClientConnection(name, loc, tok string) (*grpc.ClientConn, error) { port = "443" } - logger.Debugf("ControlTower host: %s, port: %s", host, port) + logger.Debugf("Establishing grpc connection for: %s host: %s, port: %s", + name, host, port) headers := http.Header{} headers.Set("x-tenant-id", TenantDomain()) - vetTenantMockUser := os.Getenv(controlTowerTenantEnvKey) + vetTenantMockUser := os.Getenv("VET_CONTROL_TOWER_MOCK_USER") if vetTenantMockUser != "" { headers.Set("x-mock-user", vetTenantMockUser) } diff --git a/pkg/models/models.go b/pkg/models/models.go index 6910e22..e609807 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -3,6 +3,7 @@ package models import ( "fmt" "hash/fnv" + "path/filepath" "strconv" "strings" "sync" @@ -29,17 +30,59 @@ const ( EcosystemSpdxSBOM = "SpdxSbom" ) +type ManifestSourceType string + +const ( + ManifestSourceLocal = ManifestSourceType("local") + ManifestSourceGitRepository = ManifestSourceType("git_repository") +) + +// We now have different sources from where a package +// manifest can be identified. For example, local, github, +// and may be in future within containers or archives like +// JAR. So we need to store additional internal metadata +type PackageManifestSource struct { + // The source type of this package namespace + Type ManifestSourceType + + // The namespace of the package manifest. Examples: + // - Directory when source is local + // - GitHub repo URL when source is GitHub + Namespace string + + // The namespace relative path of the package manifest + Path string + + // Explicit override the display path + DisplayPath string +} + +func (ps PackageManifestSource) GetDisplayPath() string { + switch ps.Type { + case ManifestSourceLocal: + return filepath.Join(ps.Namespace, ps.Path) + default: + return ps.DisplayPath + } +} + +func (ps PackageManifestSource) GetNamespace() string { + return ps.Namespace +} + +func (ps PackageManifestSource) GetPath() string { + return ps.Path +} + // Represents a package manifest that contains a list // of packages. Example: pom.xml, requirements.txt type PackageManifest struct { + // The source of the package manifest + Source PackageManifestSource `json:"source"` + // Filesystem path of this manifest Path string `json:"path"` - // When we scan non-path entities like Github org / repo - // then only path doesn't make sense, which is more local - // temporary file path - DisplayPath string `json:"display_path"` - // Ecosystem to interpret this manifest Ecosystem string `json:"ecosystem"` @@ -53,8 +96,30 @@ type PackageManifest struct { m sync.Mutex } +// Deprecated: Use NewPackageManifest* initializers func NewPackageManifest(path, ecosystem string) *PackageManifest { + return NewPackageManifestFromLocal(path, ecosystem) +} + +func NewPackageManifestFromLocal(path, ecosystem string) *PackageManifest { + return newPackageManifest(PackageManifestSource{ + Type: ManifestSourceLocal, + Namespace: filepath.Dir(path), + Path: filepath.Base(path), + }, path, ecosystem) +} + +func NewPackageManifestFromGitHub(repo, repoRelativePath, realPath, ecosystem string) *PackageManifest { + return newPackageManifest(PackageManifestSource{ + Type: ManifestSourceGitRepository, + Namespace: repo, + Path: repoRelativePath, + }, realPath, ecosystem) +} + +func newPackageManifest(source PackageManifestSource, path, ecosystem string) *PackageManifest { return &PackageManifest{ + Source: source, Path: path, Ecosystem: ecosystem, Packages: make([]*Package, 0), @@ -62,6 +127,16 @@ func NewPackageManifest(path, ecosystem string) *PackageManifest { } } +// Parsers usually create a package manifest from file, readers +// have the context to set the source correct. Example: GitHub reader +func (p *PackageManifest) UpdateSourceAsGitRepository(repo, repoRelativePath string) { + p.Source = PackageManifestSource{ + Type: ManifestSourceGitRepository, + Namespace: repo, + Path: repoRelativePath, + } +} + func (pm *PackageManifest) AddPackage(pkg *Package) { pm.m.Lock() defer pm.m.Unlock() @@ -74,22 +149,22 @@ func (pm *PackageManifest) AddPackage(pkg *Package) { pm.DependencyGraph.AddNode(pkg) } +func (pm *PackageManifest) GetSource() PackageManifestSource { + return pm.Source +} + func (pm *PackageManifest) GetPath() string { return pm.Path } func (pm *PackageManifest) SetDisplayPath(path string) { - pm.DisplayPath = path + pm.Source.DisplayPath = path } // GetDisplayPath returns the [DisplayPath] if available or fallsback // to [Path] func (pm *PackageManifest) GetDisplayPath() string { - if len(pm.DisplayPath) > 0 { - return pm.DisplayPath - } - - return pm.GetPath() + return pm.Source.GetDisplayPath() } // GetPackages returns the list of packages in this manifest diff --git a/pkg/readers/github_reader.go b/pkg/readers/github_reader.go index 21c7b37..665293f 100644 --- a/pkg/readers/github_reader.go +++ b/pkg/readers/github_reader.go @@ -129,7 +129,9 @@ func (p *githubReader) processTopLevelLockfiles(ctx context.Context, client *git return err } - pm.SetDisplayPath(gitUrl.GetHttpCloneURL()) + pm.UpdateSourceAsGitRepository(gitUrl.GetHttpCloneURL(), entry.GetPath()) + pm.SetDisplayPath(entry.GetURL()) + err = handler(pm, NewManifestModelReader(pm)) if err != nil { return err @@ -179,6 +181,7 @@ func (p *githubReader) processRemoteDependencyGraph(ctx context.Context, client // Override the display path because local path of the downloaded // SBOM does not actually have a meaning + manifest.UpdateSourceAsGitRepository(gitUrl.GetHttpCloneURL(), "") manifest.SetDisplayPath(gitUrl.GetHttpCloneURL()) return handler(manifest, NewManifestModelReader(manifest)) diff --git a/pkg/reporter/ci/git.go b/pkg/reporter/ci/git.go new file mode 100644 index 0000000..006e04c --- /dev/null +++ b/pkg/reporter/ci/git.go @@ -0,0 +1 @@ +package ci diff --git a/pkg/reporter/sync.go b/pkg/reporter/sync.go index 285cf77..853d917 100644 --- a/pkg/reporter/sync.go +++ b/pkg/reporter/sync.go @@ -151,6 +151,8 @@ func NewSyncReporter(config SyncReporterConfig) (Reporter, error) { trigger := controltowerv1.ToolTrigger_TOOL_TRIGGER_MANUAL source := packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED + // A multi-project sync is required for cases like GitHub org where + // we are scanning multiple repositories if !config.EnableMultiProjectSync { logger.Debugf("Report Sync: Creating tool session for project: %s, version: %s", config.ProjectName, config.ProjectVersion) @@ -197,7 +199,7 @@ func (s *syncReporter) Name() string { func (s *syncReporter) AddManifest(manifest *models.PackageManifest) { manifestSessionKey := manifest.Path if s.config.EnableMultiProjectSync && !s.sessions.hasKeyedSession(manifestSessionKey) { - projectName := manifest.GetDisplayPath() + projectName := manifest.GetSource().GetNamespace() projectVersion := "main" source := packagev1.ProjectSourceType_PROJECT_SOURCE_TYPE_UNSPECIFIED @@ -341,6 +343,7 @@ func (s *syncReporter) syncEvent(event *analyzer.AnalyzerEvent) error { logger.Warnf("unsupported check type: %s", filter.GetCheckType()) } + namespace := pkg.Manifest.GetSource().GetNamespace() req := controltowerv1.PublishPolicyViolationRequest{ ToolSession: &controltowerv1.ToolSession{ ToolSessionId: session.sessionId, @@ -348,8 +351,8 @@ func (s *syncReporter) syncEvent(event *analyzer.AnalyzerEvent) error { Manifest: &packagev1.PackageManifest{ Ecosystem: pkg.Manifest.GetControlTowerSpecEcosystem(), - Namespace: &pkg.Manifest.Path, - Name: pkg.Manifest.GetDisplayPath(), + Namespace: &namespace, + Name: pkg.Manifest.GetSource().GetPath(), }, PackageVersion: &packagev1.PackageVersion{