diff --git a/go.mod b/go.mod index fd4a457..0fa2f6b 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/minio/pkg/v3 go 1.23 require ( + github.com/cespare/xxhash/v2 v2.3.0 github.com/cheggaaa/pb v1.0.29 github.com/fatih/color v1.18.0 github.com/fatih/structs v1.1.0 github.com/go-ldap/ldap/v3 v3.4.8 - github.com/lestrrat-go/jwx v1.2.30 github.com/lestrrat-go/jwx/v2 v2.1.3 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 @@ -16,7 +16,9 @@ require ( github.com/minio/mux v1.8.2 github.com/montanaflynn/stats v0.7.1 github.com/rjeczalik/notify v0.9.3 + github.com/secure-io/sio-go v0.3.1 github.com/tinylib/msgp v1.2.5 + github.com/zeebo/xxh3 v1.0.2 go.etcd.io/etcd/client/v3 v3.5.17 golang.org/x/crypto v0.31.0 golang.org/x/sys v0.28.0 @@ -39,7 +41,6 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/kr/pretty v0.2.1 // indirect - github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect @@ -51,7 +52,6 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.61.0 // indirect @@ -61,7 +61,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.6.0 // indirect github.com/safchain/ethtool v0.5.9 // indirect - github.com/secure-io/sio-go v0.3.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/go.sum b/go.sum index ded8c6e..8a10b12 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -77,8 +79,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -87,11 +87,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx v1.2.30 h1:VKIFrmjYn0z2J51iLPadqoHIVLzvWNa1kCsTqNDHYPA= -github.com/lestrrat-go/jwx v1.2.30/go.mod h1:vMxrwFhunGZ3qddmfmEm2+uced8MSI6QFWGTKygjSzQ= github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= @@ -123,8 +120,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -180,6 +175,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/etcd/api/v3 v3.5.17 h1:cQB8eb8bxwuxOilBpMJAEo8fAONyrdXTHUNcMd8yT1w= go.etcd.io/etcd/api/v3 v3.5.17/go.mod h1:d1hvkRuXkts6PmaYk2Vrgqbv7H4ADfAKhyJqHNLJCB4= go.etcd.io/etcd/client/pkg/v3 v3.5.17 h1:XxnDXAWq2pnxqx76ljWwiQ9jylbpC4rvkAeRVOUKKVw= diff --git a/policy/policy.go b/policy/policy.go index 0b357c9..634b352 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -18,6 +18,8 @@ package policy import ( + "crypto/rand" + "encoding/binary" "encoding/json" "io" "strings" @@ -242,6 +244,11 @@ func MergePolicies(inputs ...Policy) Policy { } func (iamp *Policy) dropDuplicateStatements() { + // Select an O(N) version instead of O(N!) for more statements. + if len(iamp.Statements) > 10 { + iamp.dropDuplicateStatementsMany() + return + } dups := make(map[int]struct{}) for i := range iamp.Statements { if _, ok := dups[i]; ok { @@ -271,6 +278,30 @@ func (iamp *Policy) dropDuplicateStatements() { iamp.Statements = iamp.Statements[:c] } +func (iamp *Policy) dropDuplicateStatementsMany() { + // Calculate a hash for each. + // Drop statements with duplicate hashes. + found := make(map[[16]byte]struct{}, len(iamp.Statements)) + + // Apply a base seed + var baseSeed [8]byte + rand.Read(baseSeed[:]) + var seed uint64 + binary.LittleEndian.PutUint64(baseSeed[:], seed) + writeAt := 0 + for _, s := range iamp.Statements { + h := s.hash(seed) + if _, ok := found[h]; ok { + // duplicate, do not write. + continue + } + found[h] = struct{}{} + iamp.Statements[writeAt] = s + writeAt++ + } + iamp.Statements = iamp.Statements[:writeAt] +} + // UnmarshalJSON - decodes JSON data to Iamp. func (iamp *Policy) UnmarshalJSON(data []byte) error { // subtype to avoid recursive call to UnmarshalJSON() diff --git a/policy/policy_test.go b/policy/policy_test.go index 73651f2..08f6fe8 100644 --- a/policy/policy_test.go +++ b/policy/policy_test.go @@ -21,6 +21,7 @@ import ( "bytes" "encoding/json" "net" + "strconv" "strings" "testing" "time" @@ -1543,11 +1544,155 @@ func TestMergePolicies(t *testing.T) { }, }, }, + { + inputs: []Policy{p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3}, + expected: Policy{ + Version: DefaultVersion, + Statements: []Statement{ + NewStatement( + "", + Deny, + NewActionSet(AllAdminActions), + ResourceSet{}, + condition.NewFunctions(), + ), + NewStatement( + "", + Allow, + NewActionSet(AllActions), + NewResourceSet(NewResource("*")), + condition.NewFunctions(), + ), + NewStatement( + "", + Allow, + NewActionSet(GetBucketLocationAction), + NewResourceSet(NewResource("mybucket")), + condition.NewFunctions(), + ), + }, + }, + }, } for i, testCase := range testCases { - got := MergePolicies(testCase.inputs...) - if !got.Equals(testCase.expected) { - t.Errorf("Case %d: expected: %v, got %v", i+1, got, testCase.expected) - } + t.Run(strconv.Itoa(i), func(t *testing.T) { + got := MergePolicies(testCase.inputs...) + if !got.Equals(testCase.expected) { + t.Errorf("Case %d: expected: %v, got %v", i, testCase.expected, got) + } + }) + } +} + +func BenchmarkDedupe(b *testing.B) { + var allActions []Action + var allAdminActions []Action + for action := range supportedActions { + allActions = append(allActions, action) + } + for action := range supportedAdminActions { + allAdminActions = append(allAdminActions, Action(action)) + } + + p1 := Policy{ + Version: DefaultVersion, + Statements: []Statement{ + NewStatement( + "", + Deny, + NewActionSet(allAdminActions...), + NewResourceSet(NewResource("bucket0"), NewResource("bucket1"), NewResource("bucket2"), NewResource("bucket3"), NewResource("bucket4"), NewResource("bucket5")), + condition.NewFunctions(), + ), + NewStatement( + "", + Allow, + NewActionSet(allActions...), + NewResourceSet(NewResource("bucket0"), NewResource("bucket1"), NewResource("bucket2"), NewResource("bucket3"), NewResource("bucket4"), NewResource("bucket5")), + condition.NewFunctions(), + ), + }, + } + + // p2 is a subset of p1 + p2 := Policy{ + Version: DefaultVersion, + Statements: []Statement{ + NewStatement( + "", + Deny, + NewActionSet(allAdminActions...), + NewResourceSet(NewResource("bucket0"), NewResource("bucket1"), NewResource("bucket2"), NewResource("bucket3"), NewResource("bucket4"), NewResource("bucket5")), + condition.NewFunctions(), + ), + }, + } + + p3 := Policy{ + ID: "MyPolicyForMyBucket1", + Version: DefaultVersion, + Statements: []Statement{ + NewStatement( + "", + Allow, + NewActionSet(allActions...), + NewResourceSet(NewResource("mybucketA"), NewResource("mybucketB"), NewResource("mybucketC"), NewResource("mybucketD"), NewResource("mybucketE"), NewResource("mybucketF"), NewResource("mybucketG"), NewResource("mybucketH"), NewResource("mybucketI"), NewResource("mybucketJ"), NewResource("mybucketK"), NewResource("mybucketL"), NewResource("mybucketM"), NewResource("mybucketN"), NewResource("mybucketO"), NewResource("mybucketP"), NewResource("mybucketQ"), NewResource("mybucketR"), NewResource("mybucketS"), NewResource("mybucketS"), NewResource("mybucketU"), NewResource("mybucketV"), NewResource("mybucketX")), + condition.NewFunctions(), + ), + }, + } + + testCases := []struct { + inputs []Policy + expected Policy + }{ + { + inputs: []Policy{p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3, p1, p2, p3}, + expected: Policy{ + Version: DefaultVersion, + Statements: []Statement{ + NewStatement( + "", + Deny, + NewActionSet(allAdminActions...), + NewResourceSet(NewResource("bucket0"), NewResource("bucket1"), NewResource("bucket2"), NewResource("bucket3"), NewResource("bucket4"), NewResource("bucket5")), + condition.NewFunctions(), + ), + NewStatement( + "", + Allow, + NewActionSet(allActions...), + NewResourceSet(NewResource("bucket0"), NewResource("bucket1"), NewResource("bucket2"), NewResource("bucket3"), NewResource("bucket4"), NewResource("bucket5")), + condition.NewFunctions(), + ), + NewStatement( + "", + Allow, + NewActionSet(allActions...), + NewResourceSet(NewResource("mybucketA"), NewResource("mybucketB"), NewResource("mybucketC"), NewResource("mybucketD"), NewResource("mybucketE"), NewResource("mybucketF"), NewResource("mybucketG"), NewResource("mybucketH"), NewResource("mybucketI"), NewResource("mybucketJ"), NewResource("mybucketK"), NewResource("mybucketL"), NewResource("mybucketM"), NewResource("mybucketN"), NewResource("mybucketO"), NewResource("mybucketP"), NewResource("mybucketQ"), NewResource("mybucketR"), NewResource("mybucketS"), NewResource("mybucketS"), NewResource("mybucketU"), NewResource("mybucketV"), NewResource("mybucketX")), + condition.NewFunctions(), + ), + }, + }, + }, + } + for i, testCase := range testCases { + b.Run(strconv.Itoa(i), func(b *testing.B) { + var merged Policy + for _, p := range testCase.inputs { + if merged.Version == "" { + merged.Version = p.Version + } + for _, st := range p.Statements { + merged.Statements = append(merged.Statements, st.Clone()) + } + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + shallow := merged + shallow.dropDuplicateStatements() + } + }) } } diff --git a/policy/statement.go b/policy/statement.go index b594ac5..4897d94 100644 --- a/policy/statement.go +++ b/policy/statement.go @@ -18,9 +18,11 @@ package policy import ( + "encoding/binary" "strings" "github.com/minio/pkg/v3/policy/condition" + "github.com/zeebo/xxh3" ) // Statement - iam policy statement. @@ -203,6 +205,44 @@ func (statement Statement) Equals(st Statement) bool { return true } +// Equals checks if two statements are equal +func (statement Statement) hash(seed uint64) [16]byte { + // Order independent xor. + xorTo := func(dst *xxh3.Uint128, v xxh3.Uint128) { + dst.Lo ^= v.Lo + dst.Hi ^= v.Hi + } + // Add value with seed. + xorInt := func(dst *xxh3.Uint128, n int, seed uint64) { + var tmp [8]byte + binary.LittleEndian.PutUint64(tmp[:], uint64(n)) + xorTo(dst, xxh3.Hash128Seed(tmp[:], seed)) + } + + h := xxh3.HashString128Seed(string(statement.Effect), seed) + + xorInt(&h, len(statement.Actions), seed+1) + for action := range statement.Actions { + xorTo(&h, xxh3.HashString128Seed(string(action), seed+2)) + } + + xorInt(&h, len(statement.NotActions), seed+3) + for action := range statement.NotActions { + xorTo(&h, xxh3.HashString128Seed(string(action), seed+4)) + } + + xorInt(&h, len(statement.Resources), seed+5) + for res := range statement.Resources { + xorTo(&h, xxh3.HashString128Seed(res.Pattern+res.Type.String(), seed+6)) + } + + xorInt(&h, len(statement.Conditions), seed+7) + for _, cond := range statement.Conditions { + xorTo(&h, xxh3.HashString128Seed(cond.String(), seed+8)) + } + return h.Bytes() +} + // Clone clones Statement structure func (statement Statement) Clone() Statement { return Statement{