feat: nameserver-policy support use rule-providers and reduce domain-set memory

component/sniffer/dispatcher.go

@@ -28,8 +28,8 @@ var Dispatcher *SnifferDispatcher
 type SnifferDispatcher struct {
 	enable          bool
 	sniffers        map[sniffer.Sniffer]SnifferConfig
-	forceDomain     *trie.DomainTrie[struct{}]
-	skipSNI         *trie.DomainTrie[struct{}]
+	forceDomain     *trie.DomainSet
+	skipSNI         *trie.DomainSet
 	skipList        *cache.LruCache[string, uint8]
 	rwMux           sync.RWMutex
 	forceDnsMapping bool
@@ -37,7 +37,7 @@ type SnifferDispatcher struct {
 }
 
 func (sd *SnifferDispatcher) TCPSniff(conn *N.BufferedConn, metadata *C.Metadata) {
-	if (metadata.Host == "" && sd.parsePureIp) || sd.forceDomain.Search(metadata.Host) != nil || (metadata.DNSMode == C.DNSMapping && sd.forceDnsMapping) {
+	if (metadata.Host == "" && sd.parsePureIp) || sd.forceDomain.Has(metadata.Host) || (metadata.DNSMode == C.DNSMapping && sd.forceDnsMapping) {
 		port, err := strconv.ParseUint(metadata.DstPort, 10, 16)
 		if err != nil {
 			log.Debugln("[Sniffer] Dst port is error")
@@ -74,7 +74,7 @@ func (sd *SnifferDispatcher) TCPSniff(conn *N.BufferedConn, metadata *C.Metadata
 			log.Debugln("[Sniffer] All sniffing sniff failed with from [%s:%s] to [%s:%s]", metadata.SrcIP, metadata.SrcPort, metadata.String(), metadata.DstPort)
 			return
 		} else {
-			if sd.skipSNI.Search(host) != nil {
+			if sd.skipSNI.Has(host) {
 				log.Debugln("[Sniffer] Skip sni[%s]", host)
 				return
 			}
@@ -166,8 +166,8 @@ func NewCloseSnifferDispatcher() (*SnifferDispatcher, error) {
 	return &dispatcher, nil
 }
 
-func NewSnifferDispatcher(snifferConfig map[sniffer.Type]SnifferConfig, forceDomain *trie.DomainTrie[struct{}],
-	skipSNI *trie.DomainTrie[struct{}],
+func NewSnifferDispatcher(snifferConfig map[sniffer.Type]SnifferConfig,
+	forceDomain *trie.DomainSet, skipSNI *trie.DomainSet,
 	forceDnsMapping bool, parsePureIp bool) (*SnifferDispatcher, error) {
 	dispatcher := SnifferDispatcher{
 		enable:          true,

component/tls/config.go

@@ -15,7 +15,7 @@ import (
 )
 
 var trustCerts []*x509.Certificate
-
+var certPool *x509.CertPool
 var mutex sync.RWMutex
 var errNotMacth error = errors.New("certificate fingerprints do not match")
 
@@ -40,10 +40,20 @@ func ResetCertificate() {
 }
 
 func getCertPool() *x509.CertPool {
-	certPool, err := x509.SystemCertPool()
-	if err == nil {
-		for _, cert := range trustCerts {
-			certPool.AddCert(cert)
+	if len(trustCerts) == 0 {
+		return nil
+	}
+	if certPool == nil {
+		mutex.Lock()
+		defer mutex.Unlock()
+		if certPool != nil {
+			return certPool
+		}
+		certPool, err := x509.SystemCertPool()
+		if err == nil {
+			for _, cert := range trustCerts {
+				certPool.AddCert(cert)
+			}
 		}
 	}
 	return certPool

component/trie/domain.go

@@ -25,7 +25,7 @@ func ValidAndSplitDomain(domain string) ([]string, bool) {
 	if domain != "" && domain[len(domain)-1] == '.' {
 		return nil, false
 	}
-
+	domain=strings.ToLower(domain)
 	parts := strings.Split(domain, domainStep)
 	if len(parts) == 1 {
 		if parts[0] == "" {
@@ -123,6 +123,30 @@ func (t *DomainTrie[T]) Optimize() {
 	t.root.optimize()
 }
 
+func (t *DomainTrie[T]) Foreach(print func(domain string, data T)) {
+	for key, data := range t.root.getChildren() {
+		recursion([]string{key}, data, print)
+	}
+}
+
+func recursion[T any](items []string, node *Node[T], fn func(domain string, data T)) {
+	for key, data := range node.getChildren() {
+		newItems := append([]string{key}, items...)
+		if data != nil && data.inited {
+			domain := joinDomain(newItems)
+			if domain[0] == domainStepByte {
+				domain = complexWildcard + domain
+			}
+			fn(domain, data.Data())
+		}
+		recursion(newItems, data, fn)
+	}
+}
+
+func joinDomain(items []string) string {
+	return strings.Join(items, domainStep)
+}
+
 // New returns a new, empty Trie.
 func New[T any]() *DomainTrie[T] {
 	return &DomainTrie[T]{root: newNode[T]()}

component/trie/domain_test.go

@@ -105,3 +105,23 @@ func TestTrie_WildcardBoundary(t *testing.T) {
 
 	assert.NotNil(t, tree.Search("example.com"))
 }
+
+func TestTrie_Foreach(t *testing.T) {
+	tree := New[netip.Addr]()
+	domainList := []string{
+		"google.com",
+		"stun.*.*.*",
+		"test.*.google.com",
+		"+.baidu.com",
+		"*.baidu.com",
+		"*.*.baidu.com",
+	}
+	for _, domain := range domainList {
+		tree.Insert(domain, localIP)
+	}
+	count := 0
+	tree.Foreach(func(domain string, data netip.Addr) {
+		count++
+	})
+	assert.Equal(t, 7, count)
+}

component/trie/node.go

@@ -116,6 +116,18 @@ func (n *Node[T]) setData(data T) {
 	n.inited = true
 }
 
+func (n *Node[T]) getChildren() map[string]*Node[T] {
+	if n.childMap == nil {
+		if n.childNode != nil {
+			m := make(map[string]*Node[T])
+			m[n.childStr] = n.childNode
+			return m
+		}
+	} else {
+		return n.childMap
+	}
+	return nil
+}
 func (n *Node[T]) Data() T {
 	return n.data
 }

component/trie/set_test.go

@@ -0,0 +1,60 @@
+package trie_test
+
+import (
+	"testing"
+
+	"github.com/Dreamacro/clash/component/trie"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDomain(t *testing.T) {
+	domainSet := []string{
+		"baidu.com",
+		"google.com",
+		"www.google.com",
+		"test.a.net",
+		"test.a.oc",
+	}
+	set := trie.NewDomainSet(domainSet)
+	assert.NotNil(t, set)
+	assert.True(t, set.Has("test.a.net"))
+	assert.True(t, set.Has("google.com"))
+	assert.False(t, set.Has("www.baidu.com"))
+}
+
+func TestDomainComplexWildcard(t *testing.T) {
+	domainSet := []string{
+		"+.baidu.com",
+		"+.a.baidu.com",
+		"www.baidu.com",
+		"+.bb.baidu.com",
+		"test.a.net",
+		"test.a.oc",
+		"www.qq.com",
+	}
+	set := trie.NewDomainSet(domainSet)
+	assert.NotNil(t, set)
+	assert.False(t, set.Has("google.com"))
+	assert.True(t, set.Has("www.baidu.com"))
+	assert.True(t, set.Has("test.test.baidu.com"))
+}
+
+func TestDomainWildcard(t *testing.T) {
+	domainSet := []string{
+		"*.*.*.baidu.com",
+		"www.baidu.*",
+		"stun.*.*",
+		"*.*.qq.com",
+		"test.*.baidu.com",
+	}
+	set := trie.NewDomainSet(domainSet)
+	assert.NotNil(t, set)
+	assert.True(t, set.Has("www.baidu.com"))
+	assert.True(t, set.Has("test.test.baidu.com"))
+	assert.True(t, set.Has("test.test.qq.com"))
+	assert.True(t,set.Has("stun.ab.cd"))
+	assert.False(t, set.Has("test.baidu.com"))
+	assert.False(t,set.Has("www.google.com"))
+	assert.False(t, set.Has("test.qq.com"))
+	assert.False(t, set.Has("test.test.test.qq.com"))
+}

component/trie/sskv.go

@@ -0,0 +1,178 @@
+package trie
+
+// Package succinct provides several succinct data types.
+// Modify from https://github.com/openacid/succinct/blob/d4684c35d123f7528b14e03c24327231723db704/sskv.go
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/Dreamacro/clash/common/utils"
+	"github.com/openacid/low/bitmap"
+)
+
+const (
+	complexWildcardByte = byte('+')
+	wildcardByte        = byte('*')
+	domainStepByte      = byte('.')
+)
+
+type DomainSet struct {
+	leaves, labelBitmap []uint64
+	labels              []byte
+	ranks, selects      []int32
+}
+
+// NewDomainSet creates a new *DomainSet struct, from a slice of sorted strings.
+func NewDomainSet(keys []string) *DomainSet {
+	domainTrie := New[struct{}]()
+	for _, domain := range keys {
+		domainTrie.Insert(domain, struct{}{})
+	}
+	reserveDomains := make([]string, 0, len(keys))
+	domainTrie.Foreach(func(domain string, data struct{}) {
+		reserveDomains = append(reserveDomains, utils.Reverse(domain))
+	})
+	// ensure that the same prefix is continuous
+	// and according to the ascending sequence of length
+	sort.Strings(reserveDomains)
+	keys = reserveDomains
+	if len(keys) == 0 {
+		return nil
+	}
+	ss := &DomainSet{}
+	lIdx := 0
+
+	type qElt struct{ s, e, col int }
+	queue := []qElt{{0, len(keys), 0}}
+	for i := 0; i < len(queue); i++ {
+		elt := queue[i]
+		if elt.col == len(keys[elt.s]) {
+			elt.s++
+			// a leaf node
+			setBit(&ss.leaves, i, 1)
+		}
+
+		for j := elt.s; j < elt.e; {
+
+			frm := j
+
+			for ; j < elt.e && keys[j][elt.col] == keys[frm][elt.col]; j++ {
+			}
+			queue = append(queue, qElt{frm, j, elt.col + 1})
+			ss.labels = append(ss.labels, keys[frm][elt.col])
+			setBit(&ss.labelBitmap, lIdx, 0)
+			lIdx++
+		}
+		setBit(&ss.labelBitmap, lIdx, 1)
+		lIdx++
+	}
+
+	ss.init()
+	return ss
+}
+
+// Has query for a key and return whether it presents in the DomainSet.
+func (ss *DomainSet) Has(key string) bool {
+	if ss == nil {
+		return false
+	}
+	key = utils.Reverse(key)
+	key = strings.ToLower(key)
+	// no more labels in this node
+	// skip character matching
+	// go to next level
+	nodeId, bmIdx := 0, 0
+	type wildcardCursor struct {
+		bmIdx, index int
+	}
+	stack := make([]wildcardCursor, 0)
+	for i := 0; i < len(key); i++ {
+	RESTART:
+		c := key[i]
+		for ; ; bmIdx++ {
+			if getBit(ss.labelBitmap, bmIdx) != 0 {
+				if len(stack) > 0 {
+					cursor := stack[len(stack)-1]
+					stack = stack[0 : len(stack)-1]
+					// back wildcard and find next node
+					nextNodeId := countZeros(ss.labelBitmap, ss.ranks, cursor.bmIdx+1)
+					nextBmIdx := selectIthOne(ss.labelBitmap, ss.ranks, ss.selects, nextNodeId-1) + 1
+					j := cursor.index
+					for ; j < len(key) && key[j] != domainStepByte; j++ {
+					}
+					if j == len(key) {
+						if getBit(ss.leaves, nextNodeId) != 0 {
+							return true
+						}else {
+							goto RESTART
+						}
+					}
+					for ; ; nextBmIdx++ {
+						if ss.labels[nextBmIdx-nextNodeId] == domainStepByte {
+							bmIdx = nextBmIdx
+							nodeId = nextNodeId
+							i = j
+							goto RESTART
+						}
+					}
+				}
+				return false
+			}
+			// handle wildcard for domain
+			if ss.labels[bmIdx-nodeId] == complexWildcardByte {
+				return true
+			} else if ss.labels[bmIdx-nodeId] == wildcardByte {
+				cursor := wildcardCursor{}
+				cursor.bmIdx = bmIdx
+				cursor.index = i
+				stack = append(stack, cursor)
+			} else if ss.labels[bmIdx-nodeId] == c {
+				break
+			}
+		}
+		nodeId = countZeros(ss.labelBitmap, ss.ranks, bmIdx+1)
+		bmIdx = selectIthOne(ss.labelBitmap, ss.ranks, ss.selects, nodeId-1) + 1
+	}
+
+	return getBit(ss.leaves, nodeId) != 0
+
+}
+
+func setBit(bm *[]uint64, i int, v int) {
+	for i>>6 >= len(*bm) {
+		*bm = append(*bm, 0)
+	}
+	(*bm)[i>>6] |= uint64(v) << uint(i&63)
+}
+
+func getBit(bm []uint64, i int) uint64 {
+	return bm[i>>6] & (1 << uint(i&63))
+}
+
+// init builds pre-calculated cache to speed up rank() and select()
+func (ss *DomainSet) init() {
+	ss.selects, ss.ranks = bitmap.IndexSelect32R64(ss.labelBitmap)
+}
+
+// countZeros counts the number of "0" in a bitmap before the i-th bit(excluding
+// the i-th bit) on behalf of rank index.
+// E.g.:
+//
+//	countZeros("010010", 4) == 3
+//	//          012345
+func countZeros(bm []uint64, ranks []int32, i int) int {
+	a, _ := bitmap.Rank64(bm, ranks, int32(i))
+	return i - int(a)
+}
+
+// selectIthOne returns the index of the i-th "1" in a bitmap, on behalf of rank
+// and select indexes.
+// E.g.:
+//
+//	selectIthOne("010010", 1) == 4
+//	//            012345
+func selectIthOne(bm []uint64, ranks, selects []int32, i int) int {
+	a, _ := bitmap.Select32R64(bm, selects, ranks, int32(i))
+	return int(a)
+}