Skip to content

Commit 0bcfedb

Browse files
nikpivkinnikpivkin
authored
fix(misconf): fix caching of modules in subdirectories (#6814)
1 parent 02d5404 commit 0bcfedb

File tree

10 files changed

+263
-33
lines changed

10 files changed

+263
-33
lines changed

‎pkg/iac/scanners/terraform/parser/module_retrieval.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ type ModuleResolver interface {
1313
}
1414

1515
var defaultResolvers = []ModuleResolver{
16-
resolvers.Cache,
1716
resolvers.Local,
17+
resolvers.Cache,
1818
resolvers.Remote,
1919
resolvers.Registry,
2020
}

‎pkg/iac/scanners/terraform/parser/parser.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -232,17 +232,19 @@ func (p *Parser) Load(ctx context.Context) (*evaluator, error) {
232232
}
233233

234234
modulesMetadata, metadataPath, err := loadModuleMetadata(p.moduleFS, p.projectRoot)
235-
if err != nil {
235+
236+
if err != nil && !errors.Is(err, os.ErrNotExist) {
236237
p.debug.Log("Error loading module metadata: %s.", err)
237-
} else {
238-
p.debug.Log("Loaded module metadata for %d module(s) from '%s'.", len(modulesMetadata.Modules), metadataPath)
238+
} else if err == nil {
239+
p.debug.Log("Loaded module metadata for %d module(s) from %q.", len(modulesMetadata.Modules), metadataPath)
239240
}
240241

241242
workingDir, err := os.Getwd()
242243
if err != nil {
243244
return nil, err
244245
}
245-
p.debug.Log("Working directory for module evaluation is '%s'", workingDir)
246+
247+
p.debug.Log("Working directory for module evaluation is %q", workingDir)
246248
return newEvaluator(
247249
p.moduleFS,
248250
p,

‎pkg/iac/scanners/terraform/parser/parser_integration_test.go

+35-14
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,18 @@ func Test_DefaultRegistry(t *testing.T) {
1313
if testing.Short() {
1414
t.Skip("skipping integration test in short mode")
1515
}
16-
fs := testutil.CreateFS(t, map[string]string{
16+
17+
fsys := testutil.CreateFS(t, map[string]string{
1718
"code/test.tf": `
1819
module "registry" {
1920
source = "terraform-aws-modules/vpc/aws"
2021
}
2122
`,
2223
})
2324

24-
parser := New(fs, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
25-
if err := parser.ParseFS(context.TODO(), "code"); err != nil {
26-
t.Fatal(err)
27-
}
25+
parser := New(fsys, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
26+
require.NoError(t, parser.ParseFS(context.TODO(), "code"))
27+
2828
modules, _, err := parser.EvaluateAll(context.TODO())
2929
require.NoError(t, err)
3030
require.Len(t, modules, 2)
@@ -34,18 +34,18 @@ func Test_SpecificRegistry(t *testing.T) {
3434
if testing.Short() {
3535
t.Skip("skipping integration test in short mode")
3636
}
37-
fs := testutil.CreateFS(t, map[string]string{
37+
38+
fsys := testutil.CreateFS(t, map[string]string{
3839
"code/test.tf": `
3940
module "registry" {
4041
source = "registry.terraform.io/terraform-aws-modules/vpc/aws"
4142
}
4243
`,
4344
})
4445

45-
parser := New(fs, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
46-
if err := parser.ParseFS(context.TODO(), "code"); err != nil {
47-
t.Fatal(err)
48-
}
46+
parser := New(fsys, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
47+
require.NoError(t, parser.ParseFS(context.TODO(), "code"))
48+
4949
modules, _, err := parser.EvaluateAll(context.TODO())
5050
require.NoError(t, err)
5151
require.Len(t, modules, 2)
@@ -55,7 +55,8 @@ func Test_ModuleWithPessimisticVersionConstraint(t *testing.T) {
5555
if testing.Short() {
5656
t.Skip("skipping integration test in short mode")
5757
}
58-
fs := testutil.CreateFS(t, map[string]string{
58+
59+
fsys := testutil.CreateFS(t, map[string]string{
5960
"code/test.tf": `
6061
module "registry" {
6162
source = "registry.terraform.io/terraform-aws-modules/s3-bucket/aws"
@@ -65,10 +66,30 @@ module "registry" {
6566
`,
6667
})
6768

68-
parser := New(fs, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
69-
if err := parser.ParseFS(context.TODO(), "code"); err != nil {
70-
t.Fatal(err)
69+
parser := New(fsys, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
70+
require.NoError(t, parser.ParseFS(context.TODO(), "code"))
71+
72+
modules, _, err := parser.EvaluateAll(context.TODO())
73+
require.NoError(t, err)
74+
require.Len(t, modules, 2)
75+
}
76+
77+
func Test_ModuleInSubdir(t *testing.T) {
78+
if testing.Short() {
79+
t.Skip("skipping integration test in short mode")
7180
}
81+
82+
fsys := testutil.CreateFS(t, map[string]string{
83+
"code/test.tf": `
84+
module "object" {
85+
source = "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2"
86+
87+
}`,
88+
})
89+
90+
parser := New(fsys, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true))
91+
require.NoError(t, parser.ParseFS(context.TODO(), "code"))
92+
7293
modules, _, err := parser.EvaluateAll(context.TODO())
7394
require.NoError(t, err)
7495
require.Len(t, modules, 2)

‎pkg/iac/scanners/terraform/parser/resolvers/cache.go

+21-10
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ package resolvers
22

33
import (
44
"context"
5-
"crypto/md5" // nolint
5+
"crypto/md5" // #nosec
6+
"encoding/hex"
67
"fmt"
78
"io/fs"
89
"os"
@@ -15,16 +16,21 @@ var Cache = &cacheResolver{}
1516

1617
const tempDirName = ".aqua"
1718

18-
func locateCacheFS() (fs.FS, error) {
19-
dir, err := locateCacheDir()
19+
var defaultCacheDir = filepath.Join(os.TempDir(), tempDirName, "cache")
20+
21+
func locateCacheFS(cacheDir string) (fs.FS, error) {
22+
dir, err := locateCacheDir(cacheDir)
2023
if err != nil {
2124
return nil, err
2225
}
2326
return os.DirFS(dir), nil
2427
}
2528

26-
func locateCacheDir() (string, error) {
27-
cacheDir := filepath.Join(os.TempDir(), tempDirName, "cache")
29+
func locateCacheDir(cacheDir string) (string, error) {
30+
if cacheDir == "" {
31+
cacheDir = defaultCacheDir
32+
}
33+
2834
if err := os.MkdirAll(cacheDir, 0o750); err != nil {
2935
return "", err
3036
}
@@ -39,24 +45,29 @@ func (r *cacheResolver) Resolve(_ context.Context, _ fs.FS, opt Options) (filesy
3945
opt.Debug("Cache is disabled.")
4046
return nil, "", "", false, nil
4147
}
42-
cacheFS, err := locateCacheFS()
48+
cacheFS, err := locateCacheFS(opt.CacheDir)
4349
if err != nil {
4450
opt.Debug("No cache filesystem is available on this machine.")
4551
return nil, "", "", false, nil
4652
}
47-
key := cacheKey(opt.Source, opt.Version, opt.RelativePath)
53+
54+
src := removeSubdirFromSource(opt.Source)
55+
key := cacheKey(src, opt.Version)
56+
4857
opt.Debug("Trying to resolve: %s", key)
4958
if info, err := fs.Stat(cacheFS, filepath.ToSlash(key)); err == nil && info.IsDir() {
5059
opt.Debug("Module '%s' resolving via cache...", opt.Name)
51-
cacheDir, err := locateCacheDir()
60+
cacheDir, err := locateCacheDir(opt.CacheDir)
5261
if err != nil {
5362
return nil, "", "", true, err
5463
}
64+
5565
return os.DirFS(filepath.Join(cacheDir, key)), opt.OriginalSource, ".", true, nil
5666
}
5767
return nil, "", "", false, nil
5868
}
5969

60-
func cacheKey(source, version, relativePath string) string {
61-
return fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s:%s:%s", source, version, relativePath)))) // nolint
70+
func cacheKey(source, version string) string {
71+
hash := md5.Sum([]byte(source + ":" + version)) // #nosec
72+
return hex.EncodeToString(hash[:])
6273
}

‎pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package resolvers_test
2+
3+
import (
4+
"context"
5+
"io/fs"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
11+
"github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser/resolvers"
12+
)
13+
14+
type moduleResolver interface {
15+
Resolve(context.Context, fs.FS, resolvers.Options) (fs.FS, string, string, bool, error)
16+
}
17+
18+
func TestResolveModuleFromCache(t *testing.T) {
19+
if testing.Short() {
20+
t.Skip("skipping integration test in short mode")
21+
}
22+
23+
tests := []struct {
24+
name string
25+
opts resolvers.Options
26+
firstResolver moduleResolver
27+
}{
28+
{
29+
name: "registry",
30+
opts: resolvers.Options{
31+
Name: "bucket",
32+
Source: "terraform-aws-modules/s3-bucket/aws",
33+
Version: "4.1.2",
34+
},
35+
firstResolver: resolvers.Registry,
36+
},
37+
{
38+
name: "registry with subdir",
39+
opts: resolvers.Options{
40+
Name: "object",
41+
Source: "terraform-aws-modules/s3-bucket/aws//modules/object",
42+
Version: "4.1.2",
43+
},
44+
firstResolver: resolvers.Registry,
45+
},
46+
{
47+
name: "remote",
48+
opts: resolvers.Options{
49+
Name: "bucket",
50+
Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git?ref=v4.1.2",
51+
},
52+
firstResolver: resolvers.Remote,
53+
},
54+
{
55+
name: "remote with subdir",
56+
opts: resolvers.Options{
57+
Name: "object",
58+
Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
59+
},
60+
firstResolver: resolvers.Remote,
61+
},
62+
}
63+
64+
for _, tt := range tests {
65+
t.Run(tt.name, func(t *testing.T) {
66+
67+
tt.opts.AllowDownloads = true
68+
tt.opts.OriginalSource = tt.opts.Source
69+
tt.opts.OriginalVersion = tt.opts.Version
70+
tt.opts.CacheDir = t.TempDir()
71+
72+
fsys, _, _, applies, err := tt.firstResolver.Resolve(context.Background(), nil, tt.opts)
73+
require.NoError(t, err)
74+
assert.True(t, applies)
75+
76+
_, err = fs.Stat(fsys, "main.tf")
77+
require.NoError(t, err)
78+
79+
_, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), fsys, tt.opts)
80+
require.NoError(t, err)
81+
assert.True(t, applies)
82+
})
83+
}
84+
}
85+
86+
func TestResolveModuleFromCacheWithDifferentSubdir(t *testing.T) {
87+
if testing.Short() {
88+
t.Skip("skipping integration test in short mode")
89+
}
90+
91+
cacheDir := t.TempDir()
92+
93+
fsys, _, _, applies, err := resolvers.Remote.Resolve(context.Background(), nil, resolvers.Options{
94+
Name: "object",
95+
Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
96+
OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
97+
AllowDownloads: true,
98+
CacheDir: cacheDir,
99+
})
100+
require.NoError(t, err)
101+
assert.True(t, applies)
102+
103+
_, err = fs.Stat(fsys, "main.tf")
104+
require.NoError(t, err)
105+
106+
_, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), nil, resolvers.Options{
107+
Name: "notification",
108+
Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2",
109+
OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2",
110+
CacheDir: cacheDir,
111+
})
112+
require.NoError(t, err)
113+
assert.True(t, applies)
114+
}

‎pkg/iac/scanners/terraform/parser/resolvers/options.go

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ type Options struct {
1212
AllowDownloads bool
1313
SkipCache bool
1414
RelativePath string
15+
CacheDir string
1516
}
1617

1718
func (o *Options) hasPrefix(prefixes ...string) bool {

‎pkg/iac/scanners/terraform/parser/resolvers/registry.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option
4545
}
4646

4747
inputVersion := opt.Version
48-
source, relativePath, _ := strings.Cut(opt.Source, "//")
48+
source := removeSubdirFromSource(opt.Source)
4949
parts := strings.Split(source, "/")
5050
if len(parts) < 3 || len(parts) > 4 {
5151
return
@@ -146,7 +146,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option
146146
}
147147

148148
opt.Debug("Module '%s' resolved via registry to new source: '%s'", opt.Name, opt.Source)
149-
opt.RelativePath = relativePath
149+
150150
filesystem, prefix, downloadPath, _, err = Remote.Resolve(ctx, target, opt)
151151
if err != nil {
152152
return nil, "", "", true, err

‎pkg/iac/scanners/terraform/parser/resolvers/remote.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,11 @@ func (r *remoteResolver) Resolve(ctx context.Context, _ fs.FS, opt Options) (fil
3838
return nil, "", "", false, nil
3939
}
4040

41-
key := cacheKey(opt.OriginalSource, opt.OriginalVersion, opt.RelativePath)
41+
src := removeSubdirFromSource(opt.OriginalSource)
42+
key := cacheKey(src, opt.OriginalVersion)
4243
opt.Debug("Storing with cache key %s", key)
4344

44-
baseCacheDir, err := locateCacheDir()
45+
baseCacheDir, err := locateCacheDir(opt.CacheDir)
4546
if err != nil {
4647
return nil, "", "", true, fmt.Errorf("failed to locate cache directory: %w", err)
4748
}

‎pkg/iac/scanners/terraform/parser/resolvers/source.go

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package resolvers
2+
3+
import "strings"
4+
5+
func removeSubdirFromSource(src string) string {
6+
stop := len(src)
7+
if idx := strings.Index(src, "?"); idx > -1 {
8+
stop = idx
9+
}
10+
11+
// Calculate an offset to avoid accidentally marking the scheme
12+
// as the dir.
13+
var offset int
14+
if idx := strings.Index(src[:stop], "://"); idx > -1 {
15+
offset = idx + 3
16+
}
17+
18+
// First see if we even have an explicit subdir
19+
idx := strings.Index(src[offset:stop], "//")
20+
if idx == -1 {
21+
return src
22+
}
23+
24+
idx += offset
25+
subdir := src[idx+2:]
26+
src = src[:idx]
27+
28+
// Next, check if we have query parameters and push them onto the
29+
// URL.
30+
if idx = strings.Index(subdir, "?"); idx > -1 {
31+
query := subdir[idx:]
32+
src += query
33+
}
34+
35+
return src
36+
}

0 commit comments

Comments
 (0)