diff --git a/pkg/readers/github_org_reader.go b/pkg/readers/github_org_reader.go index d58d6cc..dcd936e 100644 --- a/pkg/readers/github_org_reader.go +++ b/pkg/readers/github_org_reader.go @@ -16,9 +16,10 @@ const ( ) type GithubOrgReaderConfig struct { - OrganizationURL string - IncludeArchived bool - MaxRepositories int + OrganizationURL string + IncludeArchived bool + MaxRepositories int + SkipDependencyGraphAPI bool } type githubOrgReader struct { @@ -129,7 +130,11 @@ func (p *githubOrgReader) handleRepositoryBatch(repositories []*github.Repositor return nil } - githubReader, err := NewGithubReader(p.client, repoUrls, "") + githubReader, err := NewGithubReader(p.client, GitHubReaderConfig{ + Urls: repoUrls, + SkipGitHubDependencyGraphAPI: p.config.SkipDependencyGraphAPI, + }) + if err != nil { return err } diff --git a/pkg/readers/github_reader.go b/pkg/readers/github_reader.go index e14013c..69e2d28 100644 --- a/pkg/readers/github_reader.go +++ b/pkg/readers/github_reader.go @@ -14,10 +14,15 @@ import ( "github.com/safedep/vet/pkg/parser" ) +type GitHubReaderConfig struct { + Urls []string + LockfileAs string + SkipGitHubDependencyGraphAPI bool +} + type githubReader struct { - client *github.Client - github_urls []string - lockfileAs string + client *github.Client + config GitHubReaderConfig } // NewGithubReader creates a [PackageManifestReader] that can be used to read @@ -25,13 +30,10 @@ type githubReader struct { // the parser auto-detects the format based on file name. This reader fails and // returns an error on first error encountered while parsing github_urls func NewGithubReader(client *github.Client, - github_urls []string, - lockfileAs string) (PackageManifestReader, error) { - + config GitHubReaderConfig) (PackageManifestReader, error) { return &githubReader{ - client: client, - github_urls: github_urls, - lockfileAs: lockfileAs, // This is unused currently + client: client, + config: config, }, nil } @@ -50,7 +52,7 @@ func (p *githubReader) EnumManifests(handler func(*models.PackageManifest, // We will not fail fast! This is because when we are scanning multiple // github urls, which we may while scanning an entire org, we want to make // as much progress as possible while logging errors - for _, github_url := range p.github_urls { + for _, github_url := range p.config.Urls { logger.Debugf("Processing Github URL: %s", github_url) gitURL, err := giturl.NewGitURL(github_url) @@ -153,6 +155,9 @@ func (p *githubReader) processTopLevelLockfiles(ctx context.Context, client *git func (p *githubReader) processRemoteDependencyGraph(ctx context.Context, client *github.Client, gitUrl giturl.IGitURL, handler func(*models.PackageManifest, PackageReader) error) error { + if p.config.SkipGitHubDependencyGraphAPI { + return errors.New("dependency graph API is disabled in the configuration") + } logger.Infof("Fetching dependency graph from %s", gitUrl.GetURL().String()) diff --git a/scan.go b/scan.go index 700f781..806ce6e 100644 --- a/scan.go +++ b/scan.go @@ -30,6 +30,7 @@ var ( githubRepoUrls []string githubOrgUrl string githubOrgMaxRepositories int + githubSkipDependencyGraphAPI bool scanExclude []string transitiveAnalysis bool transitiveDepth int @@ -97,6 +98,8 @@ func newScanCommand() *cobra.Command { "Github organization URL (Example: https://github.com/safedep)") cmd.Flags().IntVarP(&githubOrgMaxRepositories, "github-org-max-repo", "", 1000, "Maximum number of repositories to process for the Github Org") + cmd.Flags().BoolVarP(&githubSkipDependencyGraphAPI, "skip-github-dependency-graph-api", "", false, + "Do not use GitHub Dependency Graph API to fetch dependencies") cmd.Flags().StringVarP(&lockfileAs, "lockfile-as", "", "", "Parser to use for the lockfile (vet scan parsers to list)") cmd.Flags().StringVarP(&manifestType, "type", "", "", @@ -234,15 +237,20 @@ func internalStartScan() error { githubClient := githubClientBuilder() // nolint:ineffassign,staticcheck - reader, err = readers.NewGithubReader(githubClient, githubRepoUrls, lockfileAs) + reader, err = readers.NewGithubReader(githubClient, readers.GitHubReaderConfig{ + Urls: githubRepoUrls, + LockfileAs: lockfileAs, + SkipGitHubDependencyGraphAPI: githubSkipDependencyGraphAPI, + }) } else if len(githubOrgUrl) > 0 { githubClient := githubClientBuilder() // nolint:ineffassign,staticcheck reader, err = readers.NewGithubOrgReader(githubClient, &readers.GithubOrgReaderConfig{ - OrganizationURL: githubOrgUrl, - IncludeArchived: false, - MaxRepositories: githubOrgMaxRepositories, + OrganizationURL: githubOrgUrl, + IncludeArchived: false, + MaxRepositories: githubOrgMaxRepositories, + SkipDependencyGraphAPI: githubSkipDependencyGraphAPI, }) } else if len(purlSpec) > 0 { // nolint:ineffassign,staticcheck diff --git a/test/e2e_github_reader_test.go b/test/e2e_github_reader_test.go index c2af8ca..80806f9 100644 --- a/test/e2e_github_reader_test.go +++ b/test/e2e_github_reader_test.go @@ -27,10 +27,11 @@ func TestGithubReaderWithVetPublicRepository(t *testing.T) { assert.Nil(t, err, "github client creation error") - githubReader, err := readers.NewGithubReader(githubClient, []string{ - "https://github.com/safedep/vet", - "https://github.com/safedep/demo-client-java", - }, "") + githubReader, err := readers.NewGithubReader(githubClient, readers.GitHubReaderConfig{ + Urls: []string{ + "https://github.com/safedep/vet", + "https://github.com/safedep/demo-client-java", + }, LockfileAs: "", SkipGitHubDependencyGraphAPI: false}) assert.Nil(t, err, "github reader builder error") diff --git a/test/scenarios/scenario-2-vet-scan-demo-client-java.sh b/test/scenarios/scenario-2-vet-scan-demo-client-java.sh index d98dac8..68c2267 100644 --- a/test/scenarios/scenario-2-vet-scan-demo-client-java.sh +++ b/test/scenarios/scenario-2-vet-scan-demo-client-java.sh @@ -3,5 +3,5 @@ set -ex echo $( \ - $E2E_VET_BINARY scan -s --no-banner --github https://github.com/safedep/demo-client-java.git --report-summary=false --filter 'vulns.critical.exists(p, p.id == "GHSA-4wrc-f8pq-fpqp")' \ + $E2E_VET_BINARY scan -s --no-banner --github https://github.com/safedep/demo-client-java.git --report-summary=false --filter 'vulns.critical.exists(p, p.id == "GHSA-4wrc-f8pq-fpqp")' --skip-github-dependency-graph-api \ ) | grep "https://github.com/spring-projects/spring-framework"