Merge branch 'safedep:main' into main

This commit is contained in:
Hardik Nanda 2024-09-14 12:38:07 +05:30 committed by GitHub
commit 40acc58451
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 1659 additions and 407 deletions

View File

@ -27,6 +27,7 @@ CI/CD and `policy as code` as guardrails.
* [🔥 vet in action](#-vet-in-action)
* [Getting Started](#getting-started)
* [Running Scan](#running-scan)
* [Scanning Binary Artifacts](#scanning-binary-artifacts)
* [Scanning SBOM](#scanning-sbom)
* [Scanning Github Repositories](#scanning-github-repositories)
* [Scanning Github Organization](#scanning-github-organization)
@ -91,9 +92,28 @@ vet scan -D /path/to/repository
- Run `vet` to scan specific (supported) package manifests
```bash
vet scan --lockfiles /path/to/pom.xml
vet scan --lockfiles /path/to/requirements.txt
vet scan --lockfiles /path/to/package-lock.json
vet scan -M /path/to/pom.xml
vet scan -M /path/to/requirements.txt
vet scan -M /path/to/package-lock.json
```
**Note:** `--lockfiles` is generalized to `-M` or `--manifests` to support additional
types of package manifests or other artifacts in future.
#### Scanning Binary Artifacts
- Scan a Java JAR file
```bash
vet scan -M /path/to/app.jar
```
> Suitable for scanning bootable JARs with embedded dependencies
- Scan a directory with JAR files
```bash
vet scan -D /path/to/jars --type jar
```
#### Scanning SBOM
@ -101,15 +121,18 @@ vet scan --lockfiles /path/to/package-lock.json
- Scan an SBOM in [CycloneDX](https://cyclonedx.org/) format
```bash
vet scan --lockfiles /path/to/cyclonedx-sbom.json --lockfile-as bom-cyclonedx
vet scan -M /path/to/cyclonedx-sbom.json --type bom-cyclonedx
```
- Scan an SBOM in [SPDX](https://spdx.dev/) format
```bash
vet scan --lockfiles /path/to/spdx-sbom.json --lockfile-as bom-spdx
vet scan -M /path/to/spdx-sbom.json --type bom-spdx
```
**Note:** `--type` is a generalized version of `--lockfile-as` to support additional
artifact types in future.
> **Note:** SBOM scanning feature is currently in experimental stage
#### Scanning Github Repositories
@ -265,6 +288,7 @@ Refer to [CONTRIBUTING.md](CONTRIBUTING.md)
## 🔖 References
- https://github.com/google/osv-scanner
- https://github.com/anchore/syft
- https://deps.dev/
- https://securityscorecards.dev/
- https://slsa.dev/

182
go.mod
View File

@ -1,32 +1,33 @@
module github.com/safedep/vet
go 1.22
toolchain go1.22.1
go 1.22.1
require (
github.com/AlecAivazis/survey/v2 v2.3.7
github.com/CycloneDX/cyclonedx-go v0.9.0
github.com/cayleygraph/cayley v0.7.7
github.com/cayleygraph/quad v1.2.5
github.com/anchore/syft v1.11.1
github.com/cayleygraph/cayley v0.7.7-0.20240706181042-81dcd7d73e45
github.com/cayleygraph/quad v1.3.0
github.com/cli/oauth v1.0.1
github.com/deepmap/oapi-codegen v1.16.3
github.com/gofri/go-github-ratelimit v1.1.0
github.com/gojek/heimdall v5.0.2+incompatible
github.com/gojek/heimdall/v7 v7.0.3
github.com/golang/protobuf v1.5.4
github.com/google/cel-go v0.20.1
github.com/google/cel-go v0.21.0
github.com/google/go-github/v54 v54.0.0
github.com/google/osv-scanner v1.8.2
github.com/google/osv-scanner v1.8.4
github.com/jedib0t/go-pretty/v6 v6.5.9
github.com/kubescape/go-git-url v0.0.30
github.com/owenrumney/go-sarif/v2 v2.3.2
github.com/owenrumney/go-sarif/v2 v2.3.3
github.com/package-url/packageurl-go v0.1.3
github.com/safedep/dry v0.0.0-20240405050202-3b26d9386e57
github.com/safedep/dry v0.0.0-20240808054916-b31bac30d0ef
github.com/sirupsen/logrus v1.9.3
github.com/smacker/go-tree-sitter v0.0.0-20240514083259-c5d1f3f5f99e
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
github.com/spdx/tools-golang v0.5.5
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
golang.org/x/oauth2 v0.21.0
golang.org/x/oauth2 v0.23.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0
)
@ -37,52 +38,81 @@ require (
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
github.com/CloudyKit/jet/v6 v6.2.0 // indirect
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
github.com/Joker/jade v1.1.3 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/semver/v3 v3.3.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect
github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect
github.com/acobaugh/osrelease v0.1.0 // indirect
github.com/adrg/xdg v0.5.0 // indirect
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 // indirect
github.com/anchore/clio v0.0.0-20240806233806-4c50c054c508 // indirect
github.com/anchore/fangs v0.0.0-20240904151251-ac0148f53e5d // indirect
github.com/anchore/go-logger v0.0.0-20240217160628-ee28a485904f // indirect
github.com/anchore/go-macholibre v0.0.0-20240116161251-5df1434a0b50 // indirect
github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect
github.com/anchore/packageurl-go v0.1.1-0.20240507183024-848e011fc24f // indirect
github.com/anchore/stereoscope v0.0.3 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/becheran/wildmatch-go v1.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bytedance/sonic v1.11.8 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/bytedance/sonic v1.12.2 // indirect
github.com/bytedance/sonic/loader v0.2.0 // indirect
github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chainguard-dev/git-urls v1.0.2 // indirect
github.com/cloudflare/circl v1.3.8 // indirect
github.com/cloudflare/circl v1.4.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dennwc/base v1.0.0 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/dop251/goja v0.0.0-20190105122144-6d5bf35058fa // indirect
github.com/containerd/containerd v1.7.21 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/cli v27.2.0+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.2 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/dop251/goja v0.0.0-20240828124009-016eb7256539 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/facebookincubator/nvdtools v0.1.5 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/felixge/fgprof v0.9.5 // indirect
github.com/flosch/pongo2/v4 v4.0.2 // indirect
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.10.0 // indirect
github.com/github/go-spdx/v2 v2.3.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.21.0 // indirect
github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
github.com/gobuffalo/envy v1.7.1 // indirect
github.com/gobuffalo/logger v1.0.1 // indirect
github.com/gobuffalo/packd v0.3.0 // indirect
github.com/gobuffalo/packr/v2 v2.7.1 // indirect
github.com/go-playground/validator/v10 v10.22.0 // indirect
github.com/go-restruct/restruct v1.2.0-alpha // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gojek/valkyrie v0.0.0-20190210220504-8f62c1e7ba45 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2 // indirect
github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-containerregistry v0.20.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/licensecheck v0.3.1 // indirect
github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hidal-go/hidalgo v0.0.0-20190814174001-42e03f3b5eaa // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hidal-go/hidalgo v0.3.0 // indirect
github.com/iancoleman/strcase v0.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
github.com/joho/godotenv v1.3.0 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kataras/blocks v0.0.8 // indirect
@ -92,64 +122,94 @@ require (
github.com/kataras/sitemap v0.0.6 // indirect
github.com/kataras/tunnel v0.0.4 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/labstack/echo/v4 v4.12.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailgun/raymond/v2 v2.0.48 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/microcosm-cc/bluemonday v1.0.26 // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/oklog/ulid/v2 v2.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/piprate/json-gold v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.50.0 // indirect
github.com/prometheus/procfs v0.13.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pborman/indent v1.2.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/piprate/json-gold v0.5.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/profile v1.7.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/pquerna/cachecontrol v0.2.0 // indirect
github.com/prometheus/client_golang v1.20.3 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.59.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/tdewolff/minify/v2 v2.20.33 // indirect
github.com/tdewolff/parse/v2 v2.7.14 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/sylabs/squashfs v1.0.0 // indirect
github.com/tdewolff/minify/v2 v2.20.37 // indirect
github.com/tdewolff/parse/v2 v2.7.15 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/tylertreat/BoomFilters v0.0.0-20181028192813-611b3dbe80e8 // indirect
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vifraa/gopom v1.0.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect
github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yosssi/ace v0.0.5 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.23.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect
golang.org/x/arch v0.10.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.66.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)

1327
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -6,6 +6,7 @@ import (
"fmt"
"net/url"
"os"
"slices"
"strings"
jsonreportspec "github.com/safedep/vet/gen/jsonreport"
@ -146,7 +147,7 @@ func (npm *npmLockfilePoisoningAnalyzer) Analyze(manifest *models.PackageManifes
})
}
if !npmIsUrlFollowsPathConvention(lockfilePackage.Resolved, packageName) {
if !npmIsUrlFollowsPathConvention(lockfilePackage.Resolved, packageName, trustedRegistryUrls) {
logger.Debugf("npmLockfilePoisoningAnalyzer: Package [%s] resolved to an unconventional URL [%s]",
packageName, lockfilePackage.Resolved)
@ -180,14 +181,7 @@ func (npm *npmLockfilePoisoningAnalyzer) Analyze(manifest *models.PackageManifes
// Analyze the artifact URL and determine if the source is trusted
func npmIsTrustedSource(sourceUrl string, trusteUrls []string) bool {
// Go url parser cannot handle git+ssh://host:project/repo.git#commit
if len(sourceUrl) > 10 && strings.EqualFold(sourceUrl[0:10], "git+ssh://") {
if cIndex := strings.Index(sourceUrl[10:], ":"); cIndex != -1 {
sourceUrl = sourceUrl[0:10+cIndex] + "/" + sourceUrl[10+cIndex+1:]
}
}
parsedUrl, err := url.Parse(sourceUrl)
parsedUrl, err := npmParseSourceUrl(sourceUrl)
if err != nil {
logger.Errorf("npmIsTrustedSource: Failed to parse URL %s: %v",
sourceUrl, err)
@ -206,7 +200,7 @@ func npmIsTrustedSource(sourceUrl string, trusteUrls []string) bool {
// Compare with trusted URLs
for _, trusteUrl := range trusteUrls {
parsedTrustedUrl, err := url.Parse(trusteUrl)
parsedTrustedUrl, err := npmParseSourceUrl(trusteUrl)
if err != nil {
logger.Errorf("npmIsTrustedSource: Failed to parse trusted URL %s: %v",
trusteUrl, err)
@ -235,6 +229,17 @@ func npmIsTrustedSource(sourceUrl string, trusteUrls []string) bool {
return false
}
func npmParseSourceUrl(sourceUrl string) (*url.URL, error) {
// Go url parser cannot handle git+ssh://host:project/repo.git#commit
if len(sourceUrl) > 10 && strings.EqualFold(sourceUrl[0:10], "git+ssh://") {
if cIndex := strings.Index(sourceUrl[10:], ":"); cIndex != -1 {
sourceUrl = sourceUrl[0:10+cIndex] + "/" + sourceUrl[10+cIndex+1:]
}
}
return url.Parse(sourceUrl)
}
// Extract the package name from the node_modules filesystem path
func npmNodeModulesPackagePathToName(path string) string {
return utils.NpmNodeModulesPackagePathToName(path)
@ -242,9 +247,9 @@ func npmNodeModulesPackagePathToName(path string) string {
// Test if URL follows the pkg name path convention as per NPM package registry
// specification https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json
func npmIsUrlFollowsPathConvention(sourceUrl string, pkg string) bool {
func npmIsUrlFollowsPathConvention(sourceUrl string, pkg string, trustedUrls []string) bool {
// Example: https://registry.npmjs.org/express/-/express-4.17.1.tgz
parsedUrl, err := url.Parse(sourceUrl)
parsedUrl, err := npmParseSourceUrl(sourceUrl)
if err != nil {
logger.Errorf("npmIsUrlFollowsPathConvention: Failed to parse URL %s: %v",
sourceUrl, err)
@ -260,6 +265,25 @@ func npmIsUrlFollowsPathConvention(sourceUrl string, pkg string) bool {
path = path[1:]
}
acceptablePackageNames := []string{pkg}
for _, trustedUrl := range trustedUrls {
parsedTrustedUrl, err := npmParseSourceUrl(trustedUrl)
if err != nil {
logger.Errorf("npmIsUrlFollowsPathConvention: Failed to parse trusted URL %s: %v",
trustedUrl, err)
continue
}
trustedBase := parsedTrustedUrl.Path
trustedBase = strings.TrimPrefix(trustedBase, "/")
trustedBase = strings.TrimSuffix(trustedBase, "/")
acceptablePackageNames = append(acceptablePackageNames,
fmt.Sprintf("%s/%s", trustedBase, pkg))
}
// Example: @angular/core from https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz
scopedPackageName := strings.Split(path, "/-/")[0]
return scopedPackageName == pkg
return slices.Contains(acceptablePackageNames, scopedPackageName)
}

View File

@ -73,6 +73,12 @@ func TestNpmIsTrustedSource(t *testing.T) {
[]string{"https://registry.npmjs.org", "git+ssh://github.com/safedep"},
false,
},
{
"source is trusted when trusted url has a base path",
"https://registry.example.org/base/a/b/-/c.tgz",
[]string{"https://registry.example.org/base"},
true,
},
}
for _, test := range cases {
@ -85,34 +91,80 @@ func TestNpmIsTrustedSource(t *testing.T) {
func TestNpmIsUrlFollowsPathConvention(t *testing.T) {
cases := []struct {
name string
url string
pkgName string
expected bool
name string
url string
pkgName string
trustedUrls []string
expected bool
}{
{
"package name matches url path",
"https://registry.npmjs.org/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{},
true,
},
{
"package name matches scoped url path",
"https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz",
"@angular/core",
[]string{},
true,
},
{
"package name does not match scoped url path",
"https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz",
"@someother/core",
[]string{},
false,
},
{
"package path matches trusted url path",
"https://registry.npmjs.org/base/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{"https://registry.npmjs.org/base"},
true,
},
{
"package path matches trusted url path with trailing slash",
"https://registry.npmjs.org/base/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{"https://registry.npmjs.org/base/"},
true,
},
{
"package path matches trusted url path prefix",
"https://registry.npmjs.org/base/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{"https://registry.npmjs.org/base1/base2"},
false,
},
{
"package path has base without trusted url",
"https://registry.npmjs.org/base/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{},
false,
},
{
"package path matches one of the trusted url base",
"https://registry.npmjs.org/base/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{"https://registry.npmjs.org/base", "https://registry.npmjs.org/base1"},
true,
},
{
"package path matches the second trusted url base",
"https://registry.npmjs.org/base1/package-name/-/package-name-1.0.0.tgz",
"package-name",
[]string{"https://registry.npmjs.org/base", "https://registry.npmjs.org/base1"},
true,
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
actual := npmIsUrlFollowsPathConvention(test.url, test.pkgName)
actual := npmIsUrlFollowsPathConvention(test.url, test.pkgName, test.trustedUrls)
assert.Equal(t, test.expected, actual)
})
}

55
pkg/parser/jar.go Normal file
View File

@ -0,0 +1,55 @@
package parser
import (
"context"
"fmt"
"os"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/source/filesource"
"github.com/safedep/vet/pkg/common/logger"
"github.com/safedep/vet/pkg/common/purl"
"github.com/safedep/vet/pkg/models"
)
func parseJavaArchiveAsGraph(path string, config *ParserConfig) (*models.PackageManifest, error) {
fi, err := os.Stat(path)
if err != nil {
return nil, err
}
if fi.IsDir() {
return nil, fmt.Errorf("%w: %s is a directory", errUnsupportedFormat, path)
}
fs, err := filesource.NewFromPath(path)
if err != nil {
return nil, err
}
resolver, err := fs.FileResolver("")
if err != nil {
return nil, err
}
cataloger := java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig())
pkgs, _, err := cataloger.Catalog(context.Background(), resolver)
if err != nil {
return nil, err
}
manifest := models.NewPackageManifest(path, models.EcosystemMaven)
for _, pkg := range pkgs {
parsedPurl, err := purl.ParsePackageUrl(pkg.PURL)
if err != nil {
logger.Errorf("failed to parse package url: %s from jar: %s", pkg.PURL, path)
continue
}
manifest.AddPackage(&models.Package{
PackageDetails: parsedPurl.GetPackageDetails(),
})
}
return manifest, nil
}

View File

@ -1,6 +1,7 @@
package parser
import (
"errors"
"fmt"
"path/filepath"
@ -13,10 +14,16 @@ import (
)
const (
customParserTypePyWheel = "python-wheel"
customParserCycloneDXSBOM = "bom-cyclonedx"
customParserSpdxSBOM = "bom-spdx"
customParserTypeSetupPy = "setup.py"
customParserTypePyWheel = "python-wheel"
customParserCycloneDXSBOM = "bom-cyclonedx"
customParserSpdxSBOM = "bom-spdx"
customParserTypeSetupPy = "setup.py"
customParserTypeJavaArchive = "jar"
customParserTypeJavaWebAppArchive = "war"
)
var (
errUnsupportedFormat = errors.New("unsupported format")
)
// Exporting as constants for use outside this package to refer to specific
@ -71,8 +78,25 @@ type dependencyGraphParser func(lockfilePath string, config *ParserConfig) (*mod
// Maintain a map of lockfileAs to dependencyGraphParser
var dependencyGraphParsers map[string]dependencyGraphParser = map[string]dependencyGraphParser{
"package-lock.json": parseNpmPackageLockAsGraph,
customParserCycloneDXSBOM: parseSbomCycloneDxAsGraph,
"package-lock.json": parseNpmPackageLockAsGraph,
customParserCycloneDXSBOM: parseSbomCycloneDxAsGraph,
customParserTypeJavaArchive: parseJavaArchiveAsGraph,
customParserTypeJavaWebAppArchive: parseJavaArchiveAsGraph,
}
// Maintain a map of extension to lockfileAs
// Ensure that only supported extensions are added
var lockfileAsMapByExtension map[string]string = map[string]string{
"jar": customParserTypeJavaArchive,
"war": customParserTypeJavaWebAppArchive,
}
func FindLockFileAsByExtension(extension string) (string, error) {
if lockfileAs, ok := lockfileAsMapByExtension[extension]; ok {
return lockfileAs, nil
}
return "", fmt.Errorf("no format found for the extension %s", extension)
}
func List(experimental bool) []string {
@ -195,6 +219,10 @@ func (pw *parserWrapper) Ecosystem() string {
return models.EcosystemPyPI
case customParserSpdxSBOM:
return models.EcosystemSpdxSBOM
case customParserTypeJavaArchive:
return models.EcosystemMaven
case customParserTypeJavaWebAppArchive:
return models.EcosystemMaven
default:
logger.Debugf("Unsupported lockfile-as %s", pw.parseAs)
return ""

View File

@ -9,7 +9,7 @@ import (
func TestListParser(t *testing.T) {
parsers := List(false)
assert.Equal(t, 13, len(parsers))
assert.Equal(t, 15, len(parsers))
}
func TestInvalidEcosystemMapping(t *testing.T) {

70
pkg/parser/resolver.go Normal file
View File

@ -0,0 +1,70 @@
package parser
import (
"path/filepath"
"slices"
"strings"
)
type TargetScopeType string
const (
TargetScopeAll TargetScopeType = "all"
TargetScopeEmbeddedType TargetScopeType = "embedded"
TargetScopeResolveByExtension TargetScopeType = "extension"
)
// ResolveParseTarget resolves the actual path and lockfileAs
// based on the provided path and lockfileAs. It supports some
// conventions such as embedded lockfileAs in path and auto-detection
// of lockfileAs based on file extension
func ResolveParseTarget(path, lockfileAs string, scopes []TargetScopeType) (string, string, error) {
// Always use explicitly set type
if lockfileAs != "" {
return path, lockfileAs, nil
}
// We will support the format `lockfileAs:lockfile` to allow
// type override per lockfile. Use it when such an override
// is available
if resolveTargetScopeAllows(scopes, TargetScopeEmbeddedType) && strings.Contains(path, ":") {
parts := strings.Split(path, ":")
if len(parts) == 2 {
lft, err := filepath.Abs(parts[1])
if err != nil {
return "", "", err
}
return lft, parts[0], nil
}
}
// Try to resolve by extension
if resolveTargetScopeAllows(scopes, TargetScopeResolveByExtension) {
ext := strings.TrimPrefix(filepath.Ext(path), ".")
if ext != "" {
lft, err := FindLockFileAsByExtension(ext)
if err == nil {
return path, lft, nil
}
}
}
// Fallback to having parser auto-discover based on file name
return path, "", nil
}
func resolveTargetScopeAllows(scopes []TargetScopeType, required TargetScopeType) bool {
if slices.Contains(scopes, TargetScopeAll) {
return true
}
for _, s := range scopes {
if s == required {
return true
}
}
return false
}

107
pkg/parser/resolver_test.go Normal file
View File

@ -0,0 +1,107 @@
package parser
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestResolveParseTarget(t *testing.T) {
cases := []struct {
name string
scopes []TargetScopeType
path string
lockfileAs string
outputPath string
outputType string
err error
}{
{
"Explicit type",
[]TargetScopeType{TargetScopeAll},
"/a/b/c.txt",
"requirements.txt",
"/a/b/c.txt",
"requirements.txt",
nil,
},
{
"Explicit type overrides everything else",
[]TargetScopeType{TargetScopeAll},
"jar:/a/b/c.jar",
"requirements.txt",
"jar:/a/b/c.jar",
"requirements.txt",
nil,
},
{
"Path with embedded type",
[]TargetScopeType{TargetScopeAll},
"requirements.txt:/a/b/c.txt",
"",
"/a/b/c.txt",
"requirements.txt",
nil,
},
{
"Loosely embedded type in path",
[]TargetScopeType{TargetScopeAll},
"requirements.txt:/a/b/c.txt:aa",
"",
"requirements.txt:/a/b/c.txt:aa",
"",
// We do not error out because our parsers can resolve from file name
nil,
},
{
"Path with mapped extension",
[]TargetScopeType{TargetScopeAll},
"/a/b/c.jar",
"",
"/a/b/c.jar",
"jar",
nil,
},
{
"Path with unmapped extension",
[]TargetScopeType{TargetScopeAll},
"/a/b/c.py",
"",
"/a/b/c.py",
"",
nil,
},
{
"Path with mapped extension is not resolved when scope is not allowed",
[]TargetScopeType{},
"/a/b/c.jar",
"",
"/a/b/c.jar",
"",
nil,
},
{
"Path with embedded type is not resolved when scope is not allowed",
[]TargetScopeType{},
"requirements.txt:/a/b/c.txt",
"",
"requirements.txt:/a/b/c.txt",
"",
nil,
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
path, lft, err := ResolveParseTarget(test.path, test.lockfileAs, test.scopes)
if test.err != nil {
assert.ErrorContains(t, err, test.err.Error())
} else {
assert.Nil(t, err)
assert.Equal(t, test.outputPath, path)
assert.Equal(t, test.outputType, lft)
}
})
}
}

View File

@ -11,9 +11,21 @@ import (
"github.com/safedep/vet/pkg/parser"
)
type DirectoryReaderConfig struct {
// Path to enumerate
Path string
// Exclusions are regex patterns to ignore paths
Exclusions []string
// Explicitly walk for the given manifest type. If this is empty
// directory reader will automatically try to find the suitable
// parser for a given file
ManifestTypeOverride string
}
type directoryReader struct {
path string
exclusions []string
config DirectoryReaderConfig
}
// NewDirectoryReader creates a [PackageManifestReader] that can scan a directory
@ -21,11 +33,9 @@ type directoryReader struct {
// and ignore parser failure. But it will fail in case the manifest handler
// returns an error. Exclusion strings are treated as regex patterns and applied
// on the absolute file path discovered while talking the directory.
func NewDirectoryReader(path string,
exclusions []string) (PackageManifestReader, error) {
func NewDirectoryReader(config DirectoryReaderConfig) (PackageManifestReader, error) {
return &directoryReader{
path: path,
exclusions: exclusions,
config: config,
}, nil
}
@ -39,7 +49,7 @@ func (p *directoryReader) Name() string {
// with the manifest model and a default package reader implementation.
func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
PackageReader) error) error {
err := filepath.WalkDir(p.path, func(path string, info os.DirEntry, err error) error {
err := filepath.WalkDir(p.config.Path, func(path string, info os.DirEntry, err error) error {
if err != nil {
return err
}
@ -59,14 +69,24 @@ func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
return filepath.SkipDir
}
// We do not want embedded types and extension based resolution
// for directory based readers because it has higher likelihood
// of causing surprises and false positives
lockfile, lockfileAs, err := parser.ResolveParseTarget(path,
p.config.ManifestTypeOverride,
[]parser.TargetScopeType{})
if err != nil {
return err
}
// We try to find a parser by filename and try to parse it
// We do not care about error here because not all files are parseable
p, err := parser.FindParser(path, "")
p, err := parser.FindParser(lockfile, lockfileAs)
if err != nil {
return nil
}
manifest, err := p.Parse(path)
manifest, err := p.Parse(lockfile)
if err != nil {
logger.Warnf("Failed to parse: %s due to %v", path, err)
return nil
@ -81,7 +101,7 @@ func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
// TODO: Build a precompiled cache of regex patterns
func (p *directoryReader) excludedPath(path string) bool {
for _, pattern := range p.exclusions {
for _, pattern := range p.config.Exclusions {
m, err := regexp.MatchString(pattern, path)
if err != nil {
logger.Warnf("Invalid regex pattern: %s: %v", pattern, err)

View File

@ -32,7 +32,10 @@ func TestNewDirectoryReader(t *testing.T) {
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, err := NewDirectoryReader(test.path, test.exclusions)
_, err := NewDirectoryReader(DirectoryReaderConfig{
Path: test.path,
Exclusions: test.exclusions,
})
assert.Equal(t, test.err, err)
})
}
@ -114,7 +117,10 @@ func TestDirectoryReaderEnumPackages(t *testing.T) {
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
reader, _ := NewDirectoryReader(test.path, test.exclusions)
reader, _ := NewDirectoryReader(DirectoryReaderConfig{
Path: test.path,
Exclusions: test.exclusions,
})
assert.NotNil(t, reader)
manifestCount := 0
@ -192,7 +198,10 @@ func TestDirectoryReaderExcludedPath(t *testing.T) {
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
r, err := NewDirectoryReader("some-path", test.patterns)
r, err := NewDirectoryReader(DirectoryReaderConfig{
Path: "test-path",
Exclusions: test.patterns,
})
assert.Nil(t, err)
var ret bool

View File

@ -32,12 +32,18 @@ func (p *lockfileReader) Name() string {
func (p *lockfileReader) EnumManifests(handler func(*models.PackageManifest,
PackageReader) error) error {
for _, lf := range p.lockfiles {
lfParser, err := parser.FindParser(lf, p.lockfileAs)
rf, rt, err := parser.ResolveParseTarget(lf, p.lockfileAs,
[]parser.TargetScopeType{parser.TargetScopeAll})
if err != nil {
return err
}
manifest, err := lfParser.Parse(lf)
lfParser, err := parser.FindParser(rf, rt)
if err != nil {
return err
}
manifest, err := lfParser.Parse(rf)
if err != nil {
return err
}

View File

@ -5,7 +5,10 @@ import (
"fmt"
"net/http"
"strings"
"time"
"github.com/gojek/heimdall"
"github.com/gojek/heimdall/v7/hystrix"
"github.com/safedep/dry/errors"
"github.com/safedep/dry/utils"
"github.com/safedep/vet/gen/insightapi"
@ -38,8 +41,19 @@ func NewInsightBasedPackageEnricher(config InsightsBasedPackageMetaEnricherConfi
return nil
}
timeout := 5 * time.Second
backoff := heimdall.NewConstantBackoff(1*time.Second,
3*time.Second)
retriableClient := hystrix.NewClient(hystrix.WithHTTPTimeout(timeout),
hystrix.WithCommandName("insights-api-client"),
hystrix.WithMaxConcurrentRequests(10),
hystrix.WithRetryCount(3),
hystrix.WithRetrier(heimdall.NewRetrier(backoff)))
client, err := insightapi.NewClientWithResponses(config.ApiUrl,
insightapi.WithRequestEditorFn(apiKeyApplier))
insightapi.WithRequestEditorFn(apiKeyApplier),
insightapi.WithHTTPClient(retriableClient))
if err != nil {
return nil, err
}

View File

@ -110,8 +110,8 @@ func (q *cayleyQueryResult) Strings() ([]string, error) {
continue
}
n := q.store.NameOf(ref)
if n == nil {
n, err := q.store.NameOf(ref)
if (err != nil) || (n == nil) {
continue
}
@ -148,8 +148,8 @@ func (q *cayleyQueryResult) Nodes() ([]Node, error) {
continue
}
n := q.store.NameOf(ref)
if n == nil {
n, err := q.store.NameOf(ref)
if (err != nil) || (n == nil) {
continue
}

32
scan.go
View File

@ -20,6 +20,8 @@ import (
)
var (
manifests []string
manifestType string
lockfiles []string
lockfileAs string
enrich bool
@ -79,11 +81,13 @@ func newScanCommand() *cobra.Command {
cmd.Flags().BoolVarP(&enrich, "enrich", "", true,
"Enrich package metadata (almost always required) using Insights API")
cmd.Flags().StringVarP(&baseDirectory, "directory", "D", wd,
"The directory to scan for lockfiles")
"The directory to scan for package manifests")
cmd.Flags().StringArrayVarP(&scanExclude, "exclude", "", []string{},
"Name patterns to ignore while scanning a directory")
cmd.Flags().StringArrayVarP(&lockfiles, "lockfiles", "L", []string{},
"List of lockfiles to scan")
cmd.Flags().StringArrayVarP(&manifests, "manifests", "M", []string{},
"List of package manifest or archive to scan (example: jar:/tmp/foo.jar)")
cmd.Flags().StringVarP(&purlSpec, "purl", "", "",
"PURL to scan")
cmd.Flags().StringArrayVarP(&githubRepoUrls, "github", "", []string{},
@ -94,6 +98,8 @@ func newScanCommand() *cobra.Command {
"Maximum number of repositories to process for the Github Org")
cmd.Flags().StringVarP(&lockfileAs, "lockfile-as", "", "",
"Parser to use for the lockfile (vet scan parsers to list)")
cmd.Flags().StringVarP(&manifestType, "type", "", "",
"Parser to use for the artifact (vet scan parsers --experimental to list)")
cmd.Flags().BoolVarP(&transitiveAnalysis, "transitive", "", false,
"Analyze transitive dependencies")
cmd.Flags().IntVarP(&transitiveDepth, "transitive-depth", "", 2,
@ -201,12 +207,28 @@ func internalStartScan() error {
return githubClient
}
// manifestType will supersede lockfileAs and eventually deprecate it
// But for now, manifestType is backward compatible with lockfileAs
if manifestType != "" {
lockfileAs = manifestType
} else {
manifestType = lockfileAs
}
// We can easily support both directory and lockfile reader. But current UX
// contract is to support one of them at a time. Lets not break the contract
// for now and figure out UX improvement later
if len(lockfiles) > 0 {
// nolint:ineffassign,staticcheck
reader, err = readers.NewLockfileReader(lockfiles, lockfileAs)
reader, err = readers.NewLockfileReader(lockfiles, manifestType)
} else if len(manifests) > 0 {
// We will make manifestType backward compatible with lockfileAs
if manifestType == "" {
manifestType = lockfileAs
}
// nolint:ineffassign,staticcheck
reader, err = readers.NewLockfileReader(manifests, manifestType)
} else if len(githubRepoUrls) > 0 {
githubClient := githubClientBuilder()
@ -226,7 +248,11 @@ func internalStartScan() error {
reader, err = readers.NewPurlReader(purlSpec)
} else {
// nolint:ineffassign,staticcheck
reader, err = readers.NewDirectoryReader(baseDirectory, scanExclude)
reader, err = readers.NewDirectoryReader(readers.DirectoryReaderConfig{
Path: baseDirectory,
Exclusions: scanExclude,
ManifestTypeOverride: manifestType,
})
}
if err != nil {

View File

@ -14,3 +14,4 @@ bash $E2E_THIS_DIR/scenario-2-vet-scan-demo-client-java.sh
bash $E2E_THIS_DIR/scenario-3-filter-fail-fast.sh
bash $E2E_THIS_DIR/scenario-4-lfp-fail-fast.sh
bash $E2E_THIS_DIR/scenario-5-gradle-depgraph-build.sh
bash $E2E_THIS_DIR/scenario-6-manifest-flag.sh

View File

@ -0,0 +1,15 @@
#!/bin/bash
set -ex
echo $( \
$E2E_VET_BINARY scan -s --no-banner -M "$E2E_ROOT/go.mod" --report-summary=false --filter 'pkg.name == "github.com/safedep/dry"' \
) | grep "github.com/safedep/dry"
$E2E_VET_BINARY scan -s --no-banner \
-M $E2E_FIXTURES/lockfiles/nestjs-lfp-package-lock.json \
--type package-lock.json \
--report-summary=false \
--fail-fast || exit 0
exit 1