mirror of
https://github.com/safedep/vet.git
synced 2025-12-11 01:01:10 -06:00
431 lines
13 KiB
Go
431 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/google/go-github/v54/github"
|
|
"github.com/safedep/dry/utils"
|
|
"github.com/safedep/vet/internal/auth"
|
|
"github.com/safedep/vet/internal/connect"
|
|
"github.com/safedep/vet/internal/ui"
|
|
"github.com/safedep/vet/pkg/analyzer"
|
|
"github.com/safedep/vet/pkg/common/logger"
|
|
"github.com/safedep/vet/pkg/models"
|
|
"github.com/safedep/vet/pkg/parser"
|
|
"github.com/safedep/vet/pkg/readers"
|
|
"github.com/safedep/vet/pkg/reporter"
|
|
"github.com/safedep/vet/pkg/scanner"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
var (
|
|
lockfiles []string
|
|
lockfileAs string
|
|
enrich bool
|
|
baseDirectory string
|
|
purlSpec string
|
|
githubRepoUrls []string
|
|
githubOrgUrl string
|
|
githubOrgMaxRepositories int
|
|
scanExclude []string
|
|
transitiveAnalysis bool
|
|
transitiveDepth int
|
|
concurrency int
|
|
dumpJsonManifestDir string
|
|
celFilterExpression string
|
|
celFilterSuiteFile string
|
|
celFilterFailOnMatch bool
|
|
markdownReportPath string
|
|
markdownSummaryReportPath string
|
|
jsonReportPath string
|
|
consoleReport bool
|
|
summaryReport bool
|
|
summaryReportMaxAdvice int
|
|
summaryReportGroupByDirectDeps bool
|
|
csvReportPath string
|
|
silentScan bool
|
|
disableAuthVerifyBeforeScan bool
|
|
syncReport bool
|
|
syncReportProject string
|
|
graphReportDirectory string
|
|
syncReportStream string
|
|
listExperimentalParsers bool
|
|
failFast bool
|
|
trustedRegistryUrls []string
|
|
scannerExperimental bool
|
|
)
|
|
|
|
func newScanCommand() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "scan",
|
|
Short: "Scan and analyse package manifests",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
startScan()
|
|
return nil
|
|
},
|
|
}
|
|
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
cmd.Flags().BoolVarP(&silentScan, "silent", "s", false,
|
|
"Silent scan to prevent rendering UI")
|
|
cmd.Flags().BoolVarP(&failFast, "fail-fast", "", false,
|
|
"Fail fast when an issue is identified")
|
|
cmd.Flags().BoolVarP(&enrich, "enrich", "", true,
|
|
"Enrich package metadata (almost always required) using Insights API")
|
|
cmd.Flags().StringVarP(&baseDirectory, "directory", "D", wd,
|
|
"The directory to scan for lockfiles")
|
|
cmd.Flags().StringArrayVarP(&scanExclude, "exclude", "", []string{},
|
|
"Name patterns to ignore while scanning a directory")
|
|
cmd.Flags().StringArrayVarP(&lockfiles, "lockfiles", "L", []string{},
|
|
"List of lockfiles to scan")
|
|
cmd.Flags().StringVarP(&purlSpec, "purl", "", "",
|
|
"PURL to scan")
|
|
cmd.Flags().StringArrayVarP(&githubRepoUrls, "github", "", []string{},
|
|
"Github repository URL (Example: https://github.com/{org}/{repo})")
|
|
cmd.Flags().StringVarP(&githubOrgUrl, "github-org", "", "",
|
|
"Github organization URL (Example: https://github.com/safedep)")
|
|
cmd.Flags().IntVarP(&githubOrgMaxRepositories, "github-org-max-repo", "", 1000,
|
|
"Maximum number of repositories to process for the Github Org")
|
|
cmd.Flags().StringVarP(&lockfileAs, "lockfile-as", "", "",
|
|
"Parser to use for the lockfile (vet scan parsers to list)")
|
|
cmd.Flags().BoolVarP(&transitiveAnalysis, "transitive", "", false,
|
|
"Analyze transitive dependencies")
|
|
cmd.Flags().IntVarP(&transitiveDepth, "transitive-depth", "", 2,
|
|
"Analyze transitive dependencies till depth")
|
|
cmd.Flags().IntVarP(&concurrency, "concurrency", "C", 5,
|
|
"Number of concurrent analysis to run")
|
|
cmd.Flags().StringVarP(&dumpJsonManifestDir, "json-dump-dir", "", "",
|
|
"Dump enriched package manifests as JSON files to dir")
|
|
cmd.Flags().StringVarP(&celFilterExpression, "filter", "", "",
|
|
"Filter and print packages using CEL")
|
|
cmd.Flags().StringVarP(&celFilterSuiteFile, "filter-suite", "", "",
|
|
"Filter packages using CEL Filter Suite from file")
|
|
cmd.Flags().BoolVarP(&celFilterFailOnMatch, "filter-fail", "", false,
|
|
"Fail the scan if the filter match any package (security gate)")
|
|
cmd.Flags().BoolVarP(&disableAuthVerifyBeforeScan, "no-verify-auth", "", false,
|
|
"Do not verify auth token before starting scan")
|
|
cmd.Flags().StringVarP(&markdownReportPath, "report-markdown", "", "",
|
|
"Generate consolidated markdown report to file")
|
|
cmd.Flags().StringVarP(&markdownSummaryReportPath, "report-markdown-summary", "", "",
|
|
"Generate consolidate summary in markdown")
|
|
cmd.Flags().BoolVarP(&consoleReport, "report-console", "", false,
|
|
"Print a report to the console")
|
|
cmd.Flags().BoolVarP(&summaryReport, "report-summary", "", true,
|
|
"Print a summary report with actionable advice")
|
|
cmd.Flags().IntVarP(&summaryReportMaxAdvice, "report-summary-max-advice", "", 5,
|
|
"Maximum number of package risk advice to show")
|
|
cmd.Flags().BoolVarP(&summaryReportGroupByDirectDeps, "report-summary-group-by-direct-deps", "", false,
|
|
"Group summary report by direct dependencies")
|
|
cmd.Flags().StringVarP(&csvReportPath, "report-csv", "", "",
|
|
"Generate CSV report of filtered packages")
|
|
cmd.Flags().StringVarP(&jsonReportPath, "report-json", "", "",
|
|
"Generate consolidated JSON report to file (EXPERIMENTAL schema)")
|
|
cmd.Flags().StringVarP(&graphReportDirectory, "report-graph", "", "",
|
|
"Generate dependency graph (if available) as dot files to directory")
|
|
cmd.Flags().BoolVarP(&syncReport, "report-sync", "", false,
|
|
"Enable syncing report data to cloud")
|
|
cmd.Flags().StringVarP(&syncReportProject, "report-sync-project", "", "",
|
|
"Project name to use in cloud")
|
|
cmd.Flags().StringVarP(&syncReportStream, "report-sync-stream", "", "",
|
|
"Project stream name (e.g. branch) to use in cloud")
|
|
cmd.Flags().StringArrayVarP(&trustedRegistryUrls, "trusted-registry", "", []string{},
|
|
"Trusted registry URLs to use for package manifest verification")
|
|
cmd.Flags().BoolVarP(&scannerExperimental, "experimental", "", false,
|
|
"Enable experimental features in scanner")
|
|
|
|
cmd.AddCommand(listParsersCommand())
|
|
return cmd
|
|
}
|
|
|
|
func listParsersCommand() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "parsers",
|
|
Short: "List available lockfile parsers",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
fmt.Printf("Available Lockfile Parsers\n")
|
|
fmt.Printf("==========================\n\n")
|
|
|
|
for idx, p := range parser.List(listExperimentalParsers) {
|
|
fmt.Printf("[%d] %s\n", idx, p)
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
cmd.Flags().BoolVarP(&listExperimentalParsers, "experimental", "", false,
|
|
"Include experimental parsers in the list")
|
|
|
|
return cmd
|
|
}
|
|
|
|
func startScan() {
|
|
if !disableAuthVerifyBeforeScan {
|
|
err := auth.Verify(&auth.VerifyConfig{
|
|
ControlPlaneApiUrl: auth.DefaultControlPlaneApiUrl(),
|
|
})
|
|
|
|
if err != nil {
|
|
failOnError("auth/verify", fmt.Errorf("failed to verify auth token: %v. "+
|
|
"You may want to setup community mode using: vet auth configure --community", err))
|
|
}
|
|
}
|
|
|
|
if auth.CommunityMode() {
|
|
ui.PrintSuccess("Running in Community Mode")
|
|
}
|
|
|
|
failOnError("scan", internalStartScan())
|
|
}
|
|
|
|
func internalStartScan() error {
|
|
readerList := []readers.PackageManifestReader{}
|
|
var reader readers.PackageManifestReader
|
|
var err error
|
|
|
|
githubClientBuilder := func() *github.Client {
|
|
githubClient, err := connect.GetGithubClient()
|
|
if err != nil {
|
|
logger.Fatalf("Failed to build Github client: %v", err)
|
|
}
|
|
|
|
return githubClient
|
|
}
|
|
|
|
// We can easily support both directory and lockfile reader. But current UX
|
|
// contract is to support one of them at a time. Lets not break the contract
|
|
// for now and figure out UX improvement later
|
|
if len(lockfiles) > 0 {
|
|
// nolint:ineffassign,staticcheck
|
|
reader, err = readers.NewLockfileReader(lockfiles, lockfileAs)
|
|
} else if len(githubRepoUrls) > 0 {
|
|
githubClient := githubClientBuilder()
|
|
|
|
// nolint:ineffassign,staticcheck
|
|
reader, err = readers.NewGithubReader(githubClient, githubRepoUrls, lockfileAs)
|
|
} else if len(githubOrgUrl) > 0 {
|
|
githubClient := githubClientBuilder()
|
|
|
|
// nolint:ineffassign,staticcheck
|
|
reader, err = readers.NewGithubOrgReader(githubClient, &readers.GithubOrgReaderConfig{
|
|
OrganizationURL: githubOrgUrl,
|
|
IncludeArchived: false,
|
|
MaxRepositories: githubOrgMaxRepositories,
|
|
})
|
|
} else if len(purlSpec) > 0 {
|
|
// nolint:ineffassign,staticcheck
|
|
reader, err = readers.NewPurlReader(purlSpec)
|
|
} else {
|
|
// nolint:ineffassign,staticcheck
|
|
reader, err = readers.NewDirectoryReader(baseDirectory, scanExclude)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
readerList = append(readerList, reader)
|
|
|
|
// We will always use this analyzer
|
|
lfpAnalyzer, err := analyzer.NewLockfilePoisoningAnalyzer(analyzer.LockfilePoisoningAnalyzerConfig{
|
|
FailFast: failFast,
|
|
TrustedRegistryUrls: trustedRegistryUrls,
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
analyzers := []analyzer.Analyzer{lfpAnalyzer}
|
|
if !utils.IsEmptyString(dumpJsonManifestDir) {
|
|
task, err := analyzer.NewJsonDumperAnalyzer(dumpJsonManifestDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
analyzers = append(analyzers, task)
|
|
}
|
|
|
|
if !utils.IsEmptyString(celFilterExpression) {
|
|
task, err := analyzer.NewCelFilterAnalyzer(celFilterExpression,
|
|
failFast || celFilterFailOnMatch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
analyzers = append(analyzers, task)
|
|
}
|
|
|
|
if !utils.IsEmptyString(celFilterSuiteFile) {
|
|
task, err := analyzer.NewCelFilterSuiteAnalyzer(celFilterSuiteFile,
|
|
failFast || celFilterFailOnMatch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
analyzers = append(analyzers, task)
|
|
}
|
|
|
|
reporters := []reporter.Reporter{}
|
|
if consoleReport {
|
|
rp, err := reporter.NewConsoleReporter()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if summaryReport {
|
|
rp, err := reporter.NewSummaryReporter(reporter.SummaryReporterConfig{
|
|
MaxAdvice: summaryReportMaxAdvice,
|
|
GroupByDirectDependency: summaryReportGroupByDirectDeps,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if !utils.IsEmptyString(markdownReportPath) {
|
|
rp, err := reporter.NewMarkdownReportGenerator(reporter.MarkdownReportingConfig{
|
|
Path: markdownReportPath,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if !utils.IsEmptyString(markdownSummaryReportPath) {
|
|
rp, err := reporter.NewMarkdownSummaryReporter(reporter.MarkdownSummaryReporterConfig{
|
|
Path: markdownSummaryReportPath,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if !utils.IsEmptyString(jsonReportPath) {
|
|
rp, err := reporter.NewJsonReportGenerator(reporter.JsonReportingConfig{
|
|
Path: jsonReportPath,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if !utils.IsEmptyString(graphReportDirectory) {
|
|
rp, err := reporter.NewDotGraphReporter(graphReportDirectory)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if !utils.IsEmptyString(csvReportPath) {
|
|
rp, err := reporter.NewCsvReporter(reporter.CsvReportingConfig{
|
|
Path: csvReportPath,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
if syncReport {
|
|
rp, err := reporter.NewSyncReporter(reporter.SyncReporterConfig{
|
|
ProjectName: syncReportProject,
|
|
StreamName: syncReportStream,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reporters = append(reporters, rp)
|
|
}
|
|
|
|
enrichers := []scanner.PackageMetaEnricher{}
|
|
if enrich {
|
|
insightsEnricher, err := scanner.NewInsightBasedPackageEnricher(scanner.InsightsBasedPackageMetaEnricherConfig{
|
|
ApiUrl: auth.ApiUrl(),
|
|
ApiAuthKey: auth.ApiKey(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
enrichers = append(enrichers, insightsEnricher)
|
|
}
|
|
|
|
pmScanner := scanner.NewPackageManifestScanner(scanner.Config{
|
|
TransitiveAnalysis: transitiveAnalysis,
|
|
TransitiveDepth: transitiveDepth,
|
|
ConcurrentAnalyzer: concurrency,
|
|
ExcludePatterns: scanExclude,
|
|
Experimental: scannerExperimental,
|
|
}, readerList, enrichers, analyzers, reporters)
|
|
|
|
// Redirect log to files to create space for UI rendering
|
|
redirectLogToFile(logFile)
|
|
|
|
// Trackers to handle UI
|
|
var packageManifestTracker any
|
|
var packageTracker any
|
|
|
|
manifestsCount := 0
|
|
pmScanner.WithCallbacks(scanner.ScannerCallbacks{
|
|
OnStartEnumerateManifest: func() {
|
|
logger.Infof("Starting to enumerate manifests")
|
|
},
|
|
OnEnumerateManifest: func(manifest *models.PackageManifest) {
|
|
logger.Infof("Discovered a manifest at %s with %d packages",
|
|
manifest.GetDisplayPath(), manifest.GetPackagesCount())
|
|
|
|
ui.IncrementTrackerTotal(packageManifestTracker, 1)
|
|
ui.IncrementTrackerTotal(packageTracker, int64(manifest.GetPackagesCount()))
|
|
|
|
manifestsCount = manifestsCount + 1
|
|
ui.SetPinnedMessageOnProgressWriter(fmt.Sprintf("Scanning %d discovered manifest(s)",
|
|
manifestsCount))
|
|
},
|
|
OnStart: func() {
|
|
if !silentScan {
|
|
ui.StartProgressWriter()
|
|
}
|
|
|
|
packageManifestTracker = ui.TrackProgress("Scanning manifests", 0)
|
|
packageTracker = ui.TrackProgress("Scanning packages", 0)
|
|
},
|
|
OnAddTransitivePackage: func(pkg *models.Package) {
|
|
ui.IncrementTrackerTotal(packageTracker, 1)
|
|
},
|
|
OnDoneManifest: func(manifest *models.PackageManifest) {
|
|
ui.IncrementProgress(packageManifestTracker, 1)
|
|
},
|
|
OnDonePackage: func(pkg *models.Package) {
|
|
ui.IncrementProgress(packageTracker, 1)
|
|
},
|
|
BeforeFinish: func() {
|
|
ui.MarkTrackerAsDone(packageManifestTracker)
|
|
ui.MarkTrackerAsDone(packageTracker)
|
|
ui.StopProgressWriter()
|
|
},
|
|
})
|
|
|
|
return pmScanner.Start()
|
|
}
|