mirror of
https://github.com/safedep/vet.git
synced 2025-12-10 00:22:08 -06:00
Merge pull request #239 from safedep/feat/238-add-jar-scanning-support
feat: Add support for jar scanning
This commit is contained in:
Abhisek Datta
GitHub
commit
377646078f
34
README.md
34
README.md
@ -27,6 +27,7 @@ CI/CD and `policy as code` as guardrails.
|
||||
* [🔥 vet in action](#-vet-in-action)
|
||||
* [Getting Started](#getting-started)
|
||||
* [Running Scan](#running-scan)
|
||||
* [Scanning Binary Artifacts](#scanning-binary-artifacts)
|
||||
* [Scanning SBOM](#scanning-sbom)
|
||||
* [Scanning Github Repositories](#scanning-github-repositories)
|
||||
* [Scanning Github Organization](#scanning-github-organization)
|
||||
@ -91,9 +92,28 @@ vet scan -D /path/to/repository
|
||||
- Run `vet` to scan specific (supported) package manifests
|
||||
|
||||
```bash
|
||||
vet scan --lockfiles /path/to/pom.xml
|
||||
vet scan --lockfiles /path/to/requirements.txt
|
||||
vet scan --lockfiles /path/to/package-lock.json
|
||||
vet scan -M /path/to/pom.xml
|
||||
vet scan -M /path/to/requirements.txt
|
||||
vet scan -M /path/to/package-lock.json
|
||||
```
|
||||
|
||||
**Note:** `--lockfiles` is generalized to `-M` or `--manifests` to support additional
|
||||
types of package manifests or other artifacts in future.
|
||||
|
||||
#### Scanning Binary Artifacts
|
||||
|
||||
- Scan a Java JAR file
|
||||
|
||||
```bash
|
||||
vet scan -M /path/to/app.jar
|
||||
```
|
||||
|
||||
> Suitable for scanning bootable JARs with embedded dependencies
|
||||
|
||||
- Scan a directory with JAR files
|
||||
|
||||
```bash
|
||||
vet scan -D /path/to/jars --type jar
|
||||
```
|
||||
|
||||
#### Scanning SBOM
|
||||
@ -101,15 +121,18 @@ vet scan --lockfiles /path/to/package-lock.json
|
||||
- Scan an SBOM in [CycloneDX](https://cyclonedx.org/) format
|
||||
|
||||
```bash
|
||||
vet scan --lockfiles /path/to/cyclonedx-sbom.json --lockfile-as bom-cyclonedx
|
||||
vet scan -M /path/to/cyclonedx-sbom.json --type bom-cyclonedx
|
||||
```
|
||||
|
||||
- Scan an SBOM in [SPDX](https://spdx.dev/) format
|
||||
|
||||
```bash
|
||||
vet scan --lockfiles /path/to/spdx-sbom.json --lockfile-as bom-spdx
|
||||
vet scan -M /path/to/spdx-sbom.json --type bom-spdx
|
||||
```
|
||||
|
||||
**Note:** `--type` is a generalized version of `--lockfile-as` to support additional
|
||||
artifact types in future.
|
||||
|
||||
> **Note:** SBOM scanning feature is currently in experimental stage
|
||||
|
||||
#### Scanning Github Repositories
|
||||
@ -265,6 +288,7 @@ Refer to [CONTRIBUTING.md](CONTRIBUTING.md)
|
||||
## 🔖 References
|
||||
|
||||
- https://github.com/google/osv-scanner
|
||||
- https://github.com/anchore/syft
|
||||
- https://deps.dev/
|
||||
- https://securityscorecards.dev/
|
||||
- https://slsa.dev/
|
||||
|
||||
104
go.mod
104
go.mod
@ -5,6 +5,7 @@ go 1.22.1
|
||||
require (
|
||||
github.com/AlecAivazis/survey/v2 v2.3.7
|
||||
github.com/CycloneDX/cyclonedx-go v0.9.0
|
||||
github.com/anchore/syft v1.11.1
|
||||
github.com/cayleygraph/cayley v0.7.7-0.20240706181042-81dcd7d73e45
|
||||
github.com/cayleygraph/quad v1.3.0
|
||||
github.com/cli/oauth v1.0.1
|
||||
@ -15,18 +16,18 @@ require (
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/google/cel-go v0.21.0
|
||||
github.com/google/go-github/v54 v54.0.0
|
||||
github.com/google/osv-scanner v1.8.3
|
||||
github.com/google/osv-scanner v1.8.4
|
||||
github.com/jedib0t/go-pretty/v6 v6.5.9
|
||||
github.com/kubescape/go-git-url v0.0.30
|
||||
github.com/owenrumney/go-sarif/v2 v2.3.3
|
||||
github.com/package-url/packageurl-go v0.1.3
|
||||
github.com/safedep/dry v0.0.0-20240808054916-b31bac30d0ef
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/smacker/go-tree-sitter v0.0.0-20240625050157-a31a98a7c0f6
|
||||
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
|
||||
github.com/spdx/tools-golang v0.5.5
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/stretchr/testify v1.9.0
|
||||
golang.org/x/oauth2 v0.22.0
|
||||
golang.org/x/oauth2 v0.23.0
|
||||
google.golang.org/protobuf v1.34.2
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
@ -39,49 +40,79 @@ require (
|
||||
github.com/CloudyKit/jet/v6 v6.2.0 // indirect
|
||||
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
|
||||
github.com/Joker/jade v1.1.3 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.2.1 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.3.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.0.0 // indirect
|
||||
github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect
|
||||
github.com/acobaugh/osrelease v0.1.0 // indirect
|
||||
github.com/adrg/xdg v0.5.0 // indirect
|
||||
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 // indirect
|
||||
github.com/anchore/clio v0.0.0-20240806233806-4c50c054c508 // indirect
|
||||
github.com/anchore/fangs v0.0.0-20240904151251-ac0148f53e5d // indirect
|
||||
github.com/anchore/go-logger v0.0.0-20240217160628-ee28a485904f // indirect
|
||||
github.com/anchore/go-macholibre v0.0.0-20240116161251-5df1434a0b50 // indirect
|
||||
github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20240507183024-848e011fc24f // indirect
|
||||
github.com/anchore/stereoscope v0.0.3 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/becheran/wildmatch-go v1.0.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bytedance/sonic v1.12.1 // indirect
|
||||
github.com/bytedance/sonic v1.12.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.0 // indirect
|
||||
github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chainguard-dev/git-urls v1.0.2 // indirect
|
||||
github.com/cloudflare/circl v1.3.9 // indirect
|
||||
github.com/cloudflare/circl v1.4.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/containerd/containerd v1.7.21 // indirect
|
||||
github.com/containerd/errdefs v0.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dlclark/regexp2 v1.11.4 // indirect
|
||||
github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe // indirect
|
||||
github.com/docker/cli v27.2.0+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.8.2 // indirect
|
||||
github.com/docker/go-connections v0.5.0 // indirect
|
||||
github.com/dop251/goja v0.0.0-20240828124009-016eb7256539 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
|
||||
github.com/facebookincubator/nvdtools v0.1.5 // indirect
|
||||
github.com/fatih/structs v1.1.0 // indirect
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/flosch/pongo2/v4 v4.0.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gin-gonic/gin v1.10.0 // indirect
|
||||
github.com/github/go-spdx/v2 v2.3.1 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.22.0 // indirect
|
||||
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/gojek/valkyrie v0.0.0-20190210220504-8f62c1e7ba45 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/google/go-containerregistry v0.20.2 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/google/licensecheck v0.3.1 // indirect
|
||||
github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/hidal-go/hidalgo v0.3.0 // indirect
|
||||
github.com/iancoleman/strcase v0.3.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/iris-contrib/schema v0.0.6 // indirect
|
||||
github.com/jinzhu/copier v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kataras/blocks v0.0.8 // indirect
|
||||
@ -93,63 +124,92 @@ require (
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/labstack/echo/v4 v4.12.0 // indirect
|
||||
github.com/labstack/gommon v0.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailgun/raymond/v2 v2.0.48 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/mholt/archiver/v3 v3.5.1 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moby/sys/mountinfo v0.7.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nwaples/rardecode v1.1.3 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/pborman/indent v1.2.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/piprate/json-gold v0.5.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pkg/profile v1.7.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/pquerna/cachecontrol v0.2.0 // indirect
|
||||
github.com/prometheus/client_golang v1.19.1 // indirect
|
||||
github.com/prometheus/client_golang v1.20.3 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/common v0.59.1 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.6.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
|
||||
github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.7.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.19.0 // indirect
|
||||
github.com/stoewer/go-strcase v1.3.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/sylabs/squashfs v1.0.0 // indirect
|
||||
github.com/tdewolff/minify/v2 v2.20.37 // indirect
|
||||
github.com/tdewolff/parse/v2 v2.7.15 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/ulikunitz/xz v0.5.12 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
github.com/vifraa/gopom v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect
|
||||
github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0 // indirect
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yosssi/ace v0.0.5 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/arch v0.9.0 // indirect
|
||||
golang.org/x/crypto v0.26.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
|
||||
golang.org/x/mod v0.20.0 // indirect
|
||||
golang.org/x/net v0.28.0 // indirect
|
||||
golang.org/x/sys v0.24.0 // indirect
|
||||
golang.org/x/term v0.23.0 // indirect
|
||||
golang.org/x/text v0.17.0 // indirect
|
||||
golang.org/x/arch v0.10.0 // indirect
|
||||
golang.org/x/crypto v0.27.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect
|
||||
golang.org/x/mod v0.21.0 // indirect
|
||||
golang.org/x/net v0.29.0 // indirect
|
||||
golang.org/x/sys v0.25.0 // indirect
|
||||
golang.org/x/term v0.24.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
golang.org/x/time v0.6.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
|
||||
google.golang.org/grpc v1.66.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
|
||||
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
|
||||
sigs.k8s.io/yaml v1.4.0 // indirect
|
||||
)
|
||||
|
||||
55
pkg/parser/jar.go
Normal file
55
pkg/parser/jar.go
Normal file
@ -0,0 +1,55 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/source/filesource"
|
||||
"github.com/safedep/vet/pkg/common/logger"
|
||||
"github.com/safedep/vet/pkg/common/purl"
|
||||
"github.com/safedep/vet/pkg/models"
|
||||
)
|
||||
|
||||
func parseJavaArchiveAsGraph(path string, config *ParserConfig) (*models.PackageManifest, error) {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.IsDir() {
|
||||
return nil, fmt.Errorf("%w: %s is a directory", errUnsupportedFormat, path)
|
||||
}
|
||||
|
||||
fs, err := filesource.NewFromPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resolver, err := fs.FileResolver("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cataloger := java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig())
|
||||
pkgs, _, err := cataloger.Catalog(context.Background(), resolver)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifest := models.NewPackageManifest(path, models.EcosystemMaven)
|
||||
for _, pkg := range pkgs {
|
||||
parsedPurl, err := purl.ParsePackageUrl(pkg.PURL)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to parse package url: %s from jar: %s", pkg.PURL, path)
|
||||
continue
|
||||
}
|
||||
|
||||
manifest.AddPackage(&models.Package{
|
||||
PackageDetails: parsedPurl.GetPackageDetails(),
|
||||
})
|
||||
}
|
||||
|
||||
return manifest, nil
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
@ -13,10 +14,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
customParserTypePyWheel = "python-wheel"
|
||||
customParserCycloneDXSBOM = "bom-cyclonedx"
|
||||
customParserSpdxSBOM = "bom-spdx"
|
||||
customParserTypeSetupPy = "setup.py"
|
||||
customParserTypePyWheel = "python-wheel"
|
||||
customParserCycloneDXSBOM = "bom-cyclonedx"
|
||||
customParserSpdxSBOM = "bom-spdx"
|
||||
customParserTypeSetupPy = "setup.py"
|
||||
customParserTypeJavaArchive = "jar"
|
||||
customParserTypeJavaWebAppArchive = "war"
|
||||
)
|
||||
|
||||
var (
|
||||
errUnsupportedFormat = errors.New("unsupported format")
|
||||
)
|
||||
|
||||
// Exporting as constants for use outside this package to refer to specific
|
||||
@ -71,8 +78,25 @@ type dependencyGraphParser func(lockfilePath string, config *ParserConfig) (*mod
|
||||
|
||||
// Maintain a map of lockfileAs to dependencyGraphParser
|
||||
var dependencyGraphParsers map[string]dependencyGraphParser = map[string]dependencyGraphParser{
|
||||
"package-lock.json": parseNpmPackageLockAsGraph,
|
||||
customParserCycloneDXSBOM: parseSbomCycloneDxAsGraph,
|
||||
"package-lock.json": parseNpmPackageLockAsGraph,
|
||||
customParserCycloneDXSBOM: parseSbomCycloneDxAsGraph,
|
||||
customParserTypeJavaArchive: parseJavaArchiveAsGraph,
|
||||
customParserTypeJavaWebAppArchive: parseJavaArchiveAsGraph,
|
||||
}
|
||||
|
||||
// Maintain a map of extension to lockfileAs
|
||||
// Ensure that only supported extensions are added
|
||||
var lockfileAsMapByExtension map[string]string = map[string]string{
|
||||
"jar": customParserTypeJavaArchive,
|
||||
"war": customParserTypeJavaWebAppArchive,
|
||||
}
|
||||
|
||||
func FindLockFileAsByExtension(extension string) (string, error) {
|
||||
if lockfileAs, ok := lockfileAsMapByExtension[extension]; ok {
|
||||
return lockfileAs, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no format found for the extension %s", extension)
|
||||
}
|
||||
|
||||
func List(experimental bool) []string {
|
||||
@ -195,6 +219,10 @@ func (pw *parserWrapper) Ecosystem() string {
|
||||
return models.EcosystemPyPI
|
||||
case customParserSpdxSBOM:
|
||||
return models.EcosystemSpdxSBOM
|
||||
case customParserTypeJavaArchive:
|
||||
return models.EcosystemMaven
|
||||
case customParserTypeJavaWebAppArchive:
|
||||
return models.EcosystemMaven
|
||||
default:
|
||||
logger.Debugf("Unsupported lockfile-as %s", pw.parseAs)
|
||||
return ""
|
||||
|
||||
@ -9,7 +9,7 @@ import (
|
||||
|
||||
func TestListParser(t *testing.T) {
|
||||
parsers := List(false)
|
||||
assert.Equal(t, 13, len(parsers))
|
||||
assert.Equal(t, 15, len(parsers))
|
||||
}
|
||||
|
||||
func TestInvalidEcosystemMapping(t *testing.T) {
|
||||
|
||||
70
pkg/parser/resolver.go
Normal file
70
pkg/parser/resolver.go
Normal file
@ -0,0 +1,70 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type TargetScopeType string
|
||||
|
||||
const (
|
||||
TargetScopeAll TargetScopeType = "all"
|
||||
TargetScopeEmbeddedType TargetScopeType = "embedded"
|
||||
TargetScopeResolveByExtension TargetScopeType = "extension"
|
||||
)
|
||||
|
||||
// ResolveParseTarget resolves the actual path and lockfileAs
|
||||
// based on the provided path and lockfileAs. It supports some
|
||||
// conventions such as embedded lockfileAs in path and auto-detection
|
||||
// of lockfileAs based on file extension
|
||||
func ResolveParseTarget(path, lockfileAs string, scopes []TargetScopeType) (string, string, error) {
|
||||
// Always use explicitly set type
|
||||
if lockfileAs != "" {
|
||||
return path, lockfileAs, nil
|
||||
}
|
||||
|
||||
// We will support the format `lockfileAs:lockfile` to allow
|
||||
// type override per lockfile. Use it when such an override
|
||||
// is available
|
||||
if resolveTargetScopeAllows(scopes, TargetScopeEmbeddedType) && strings.Contains(path, ":") {
|
||||
parts := strings.Split(path, ":")
|
||||
if len(parts) == 2 {
|
||||
lft, err := filepath.Abs(parts[1])
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return lft, parts[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
// Try to resolve by extension
|
||||
if resolveTargetScopeAllows(scopes, TargetScopeResolveByExtension) {
|
||||
ext := strings.TrimPrefix(filepath.Ext(path), ".")
|
||||
if ext != "" {
|
||||
lft, err := FindLockFileAsByExtension(ext)
|
||||
if err == nil {
|
||||
return path, lft, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to having parser auto-discover based on file name
|
||||
return path, "", nil
|
||||
|
||||
}
|
||||
|
||||
func resolveTargetScopeAllows(scopes []TargetScopeType, required TargetScopeType) bool {
|
||||
if slices.Contains(scopes, TargetScopeAll) {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, s := range scopes {
|
||||
if s == required {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
107
pkg/parser/resolver_test.go
Normal file
107
pkg/parser/resolver_test.go
Normal file
@ -0,0 +1,107 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestResolveParseTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
scopes []TargetScopeType
|
||||
path string
|
||||
lockfileAs string
|
||||
outputPath string
|
||||
outputType string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
"Explicit type",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"/a/b/c.txt",
|
||||
"requirements.txt",
|
||||
"/a/b/c.txt",
|
||||
"requirements.txt",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Explicit type overrides everything else",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"jar:/a/b/c.jar",
|
||||
"requirements.txt",
|
||||
"jar:/a/b/c.jar",
|
||||
"requirements.txt",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Path with embedded type",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"requirements.txt:/a/b/c.txt",
|
||||
"",
|
||||
"/a/b/c.txt",
|
||||
"requirements.txt",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Loosely embedded type in path",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"requirements.txt:/a/b/c.txt:aa",
|
||||
"",
|
||||
"requirements.txt:/a/b/c.txt:aa",
|
||||
"",
|
||||
|
||||
// We do not error out because our parsers can resolve from file name
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Path with mapped extension",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"/a/b/c.jar",
|
||||
"",
|
||||
"/a/b/c.jar",
|
||||
"jar",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Path with unmapped extension",
|
||||
[]TargetScopeType{TargetScopeAll},
|
||||
"/a/b/c.py",
|
||||
"",
|
||||
"/a/b/c.py",
|
||||
"",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Path with mapped extension is not resolved when scope is not allowed",
|
||||
[]TargetScopeType{},
|
||||
"/a/b/c.jar",
|
||||
"",
|
||||
"/a/b/c.jar",
|
||||
"",
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"Path with embedded type is not resolved when scope is not allowed",
|
||||
[]TargetScopeType{},
|
||||
"requirements.txt:/a/b/c.txt",
|
||||
"",
|
||||
"requirements.txt:/a/b/c.txt",
|
||||
"",
|
||||
nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path, lft, err := ResolveParseTarget(test.path, test.lockfileAs, test.scopes)
|
||||
if test.err != nil {
|
||||
assert.ErrorContains(t, err, test.err.Error())
|
||||
} else {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, test.outputPath, path)
|
||||
assert.Equal(t, test.outputType, lft)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -11,9 +11,21 @@ import (
|
||||
"github.com/safedep/vet/pkg/parser"
|
||||
)
|
||||
|
||||
type DirectoryReaderConfig struct {
|
||||
// Path to enumerate
|
||||
Path string
|
||||
|
||||
// Exclusions are regex patterns to ignore paths
|
||||
Exclusions []string
|
||||
|
||||
// Explicitly walk for the given manifest type. If this is empty
|
||||
// directory reader will automatically try to find the suitable
|
||||
// parser for a given file
|
||||
ManifestTypeOverride string
|
||||
}
|
||||
|
||||
type directoryReader struct {
|
||||
path string
|
||||
exclusions []string
|
||||
config DirectoryReaderConfig
|
||||
}
|
||||
|
||||
// NewDirectoryReader creates a [PackageManifestReader] that can scan a directory
|
||||
@ -21,11 +33,9 @@ type directoryReader struct {
|
||||
// and ignore parser failure. But it will fail in case the manifest handler
|
||||
// returns an error. Exclusion strings are treated as regex patterns and applied
|
||||
// on the absolute file path discovered while talking the directory.
|
||||
func NewDirectoryReader(path string,
|
||||
exclusions []string) (PackageManifestReader, error) {
|
||||
func NewDirectoryReader(config DirectoryReaderConfig) (PackageManifestReader, error) {
|
||||
return &directoryReader{
|
||||
path: path,
|
||||
exclusions: exclusions,
|
||||
config: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -39,7 +49,7 @@ func (p *directoryReader) Name() string {
|
||||
// with the manifest model and a default package reader implementation.
|
||||
func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
|
||||
PackageReader) error) error {
|
||||
err := filepath.WalkDir(p.path, func(path string, info os.DirEntry, err error) error {
|
||||
err := filepath.WalkDir(p.config.Path, func(path string, info os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -59,14 +69,24 @@ func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
// We do not want embedded types and extension based resolution
|
||||
// for directory based readers because it has higher likelihood
|
||||
// of causing surprises and false positives
|
||||
lockfile, lockfileAs, err := parser.ResolveParseTarget(path,
|
||||
p.config.ManifestTypeOverride,
|
||||
[]parser.TargetScopeType{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We try to find a parser by filename and try to parse it
|
||||
// We do not care about error here because not all files are parseable
|
||||
p, err := parser.FindParser(path, "")
|
||||
p, err := parser.FindParser(lockfile, lockfileAs)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
manifest, err := p.Parse(path)
|
||||
manifest, err := p.Parse(lockfile)
|
||||
if err != nil {
|
||||
logger.Warnf("Failed to parse: %s due to %v", path, err)
|
||||
return nil
|
||||
@ -81,7 +101,7 @@ func (p *directoryReader) EnumManifests(handler func(*models.PackageManifest,
|
||||
|
||||
// TODO: Build a precompiled cache of regex patterns
|
||||
func (p *directoryReader) excludedPath(path string) bool {
|
||||
for _, pattern := range p.exclusions {
|
||||
for _, pattern := range p.config.Exclusions {
|
||||
m, err := regexp.MatchString(pattern, path)
|
||||
if err != nil {
|
||||
logger.Warnf("Invalid regex pattern: %s: %v", pattern, err)
|
||||
|
||||
@ -32,7 +32,10 @@ func TestNewDirectoryReader(t *testing.T) {
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := NewDirectoryReader(test.path, test.exclusions)
|
||||
_, err := NewDirectoryReader(DirectoryReaderConfig{
|
||||
Path: test.path,
|
||||
Exclusions: test.exclusions,
|
||||
})
|
||||
assert.Equal(t, test.err, err)
|
||||
})
|
||||
}
|
||||
@ -114,7 +117,10 @@ func TestDirectoryReaderEnumPackages(t *testing.T) {
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
reader, _ := NewDirectoryReader(test.path, test.exclusions)
|
||||
reader, _ := NewDirectoryReader(DirectoryReaderConfig{
|
||||
Path: test.path,
|
||||
Exclusions: test.exclusions,
|
||||
})
|
||||
assert.NotNil(t, reader)
|
||||
|
||||
manifestCount := 0
|
||||
@ -192,7 +198,10 @@ func TestDirectoryReaderExcludedPath(t *testing.T) {
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
r, err := NewDirectoryReader("some-path", test.patterns)
|
||||
r, err := NewDirectoryReader(DirectoryReaderConfig{
|
||||
Path: "test-path",
|
||||
Exclusions: test.patterns,
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
|
||||
var ret bool
|
||||
|
||||
@ -32,12 +32,18 @@ func (p *lockfileReader) Name() string {
|
||||
func (p *lockfileReader) EnumManifests(handler func(*models.PackageManifest,
|
||||
PackageReader) error) error {
|
||||
for _, lf := range p.lockfiles {
|
||||
lfParser, err := parser.FindParser(lf, p.lockfileAs)
|
||||
rf, rt, err := parser.ResolveParseTarget(lf, p.lockfileAs,
|
||||
[]parser.TargetScopeType{parser.TargetScopeAll})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := lfParser.Parse(lf)
|
||||
lfParser, err := parser.FindParser(rf, rt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := lfParser.Parse(rf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
32
scan.go
32
scan.go
@ -20,6 +20,8 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
manifests []string
|
||||
manifestType string
|
||||
lockfiles []string
|
||||
lockfileAs string
|
||||
enrich bool
|
||||
@ -79,11 +81,13 @@ func newScanCommand() *cobra.Command {
|
||||
cmd.Flags().BoolVarP(&enrich, "enrich", "", true,
|
||||
"Enrich package metadata (almost always required) using Insights API")
|
||||
cmd.Flags().StringVarP(&baseDirectory, "directory", "D", wd,
|
||||
"The directory to scan for lockfiles")
|
||||
"The directory to scan for package manifests")
|
||||
cmd.Flags().StringArrayVarP(&scanExclude, "exclude", "", []string{},
|
||||
"Name patterns to ignore while scanning a directory")
|
||||
cmd.Flags().StringArrayVarP(&lockfiles, "lockfiles", "L", []string{},
|
||||
"List of lockfiles to scan")
|
||||
cmd.Flags().StringArrayVarP(&manifests, "manifests", "M", []string{},
|
||||
"List of package manifest or archive to scan (example: jar:/tmp/foo.jar)")
|
||||
cmd.Flags().StringVarP(&purlSpec, "purl", "", "",
|
||||
"PURL to scan")
|
||||
cmd.Flags().StringArrayVarP(&githubRepoUrls, "github", "", []string{},
|
||||
@ -94,6 +98,8 @@ func newScanCommand() *cobra.Command {
|
||||
"Maximum number of repositories to process for the Github Org")
|
||||
cmd.Flags().StringVarP(&lockfileAs, "lockfile-as", "", "",
|
||||
"Parser to use for the lockfile (vet scan parsers to list)")
|
||||
cmd.Flags().StringVarP(&manifestType, "type", "", "",
|
||||
"Parser to use for the artifact (vet scan parsers --experimental to list)")
|
||||
cmd.Flags().BoolVarP(&transitiveAnalysis, "transitive", "", false,
|
||||
"Analyze transitive dependencies")
|
||||
cmd.Flags().IntVarP(&transitiveDepth, "transitive-depth", "", 2,
|
||||
@ -201,12 +207,28 @@ func internalStartScan() error {
|
||||
return githubClient
|
||||
}
|
||||
|
||||
// manifestType will supersede lockfileAs and eventually deprecate it
|
||||
// But for now, manifestType is backward compatible with lockfileAs
|
||||
if manifestType != "" {
|
||||
lockfileAs = manifestType
|
||||
} else {
|
||||
manifestType = lockfileAs
|
||||
}
|
||||
|
||||
// We can easily support both directory and lockfile reader. But current UX
|
||||
// contract is to support one of them at a time. Lets not break the contract
|
||||
// for now and figure out UX improvement later
|
||||
if len(lockfiles) > 0 {
|
||||
// nolint:ineffassign,staticcheck
|
||||
reader, err = readers.NewLockfileReader(lockfiles, lockfileAs)
|
||||
reader, err = readers.NewLockfileReader(lockfiles, manifestType)
|
||||
} else if len(manifests) > 0 {
|
||||
// We will make manifestType backward compatible with lockfileAs
|
||||
if manifestType == "" {
|
||||
manifestType = lockfileAs
|
||||
}
|
||||
|
||||
// nolint:ineffassign,staticcheck
|
||||
reader, err = readers.NewLockfileReader(manifests, manifestType)
|
||||
} else if len(githubRepoUrls) > 0 {
|
||||
githubClient := githubClientBuilder()
|
||||
|
||||
@ -226,7 +248,11 @@ func internalStartScan() error {
|
||||
reader, err = readers.NewPurlReader(purlSpec)
|
||||
} else {
|
||||
// nolint:ineffassign,staticcheck
|
||||
reader, err = readers.NewDirectoryReader(baseDirectory, scanExclude)
|
||||
reader, err = readers.NewDirectoryReader(readers.DirectoryReaderConfig{
|
||||
Path: baseDirectory,
|
||||
Exclusions: scanExclude,
|
||||
ManifestTypeOverride: manifestType,
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@ -14,3 +14,4 @@ bash $E2E_THIS_DIR/scenario-2-vet-scan-demo-client-java.sh
|
||||
bash $E2E_THIS_DIR/scenario-3-filter-fail-fast.sh
|
||||
bash $E2E_THIS_DIR/scenario-4-lfp-fail-fast.sh
|
||||
bash $E2E_THIS_DIR/scenario-5-gradle-depgraph-build.sh
|
||||
bash $E2E_THIS_DIR/scenario-6-manifest-flag.sh
|
||||
|
||||
15
test/scenarios/scenario-6-manifest-flag.sh
Normal file
15
test/scenarios/scenario-6-manifest-flag.sh
Normal file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
echo $( \
|
||||
$E2E_VET_BINARY scan -s --no-banner -M "$E2E_ROOT/go.mod" --report-summary=false --filter 'pkg.name == "github.com/safedep/dry"' \
|
||||
) | grep "github.com/safedep/dry"
|
||||
|
||||
$E2E_VET_BINARY scan -s --no-banner \
|
||||
-M $E2E_FIXTURES/lockfiles/nestjs-lfp-package-lock.json \
|
||||
--type package-lock.json \
|
||||
--report-summary=false \
|
||||
--fail-fast || exit 0
|
||||
|
||||
exit 1
|
||||
Loading…
x
Reference in New Issue
Block a user