diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..67492579 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: go +go_import_path: github.com/golang/groupcache + +os: linux +dist: trusty +sudo: false + +script: + - go test ./... + +go: + - 1.9.x + - 1.10.x + - 1.11.x + - master + +cache: + directories: + - $GOPATH/pkg diff --git a/README.md b/README.md index 70c29da1..e4337ad3 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ ## Summary -groupcache is a caching and cache-filling library, intended as a -replacement for memcached in many cases. +groupcache is a distributed caching and cache-filling library, intended as a +replacement for a pool of memcached nodes in many cases. For API docs and examples, see http://godoc.org/github.com/golang/groupcache @@ -17,7 +17,8 @@ For API docs and examples, see http://godoc.org/github.com/golang/groupcache * does not require running a separate set of servers, thus massively reducing deployment/configuration pain. groupcache is a client - library as well as a server. It connects to its own peers. + library as well as a server. It connects to its own peers, forming + a distributed cache. * comes with a cache filling mechanism. Whereas memcached just says "Sorry, cache miss", often resulting in a thundering herd of diff --git a/consistenthash/consistenthash.go b/consistenthash/consistenthash.go index a9c56f07..da139094 100644 --- a/consistenthash/consistenthash.go +++ b/consistenthash/consistenthash.go @@ -44,12 +44,12 @@ func New(replicas int, fn Hash) *Map { return m } -// Returns true if there are no items available. +// IsEmpty returns true if there are no items available. func (m *Map) IsEmpty() bool { return len(m.keys) == 0 } -// Adds some keys to the hash. +// Add adds some keys to the hash. func (m *Map) Add(keys ...string) { for _, key := range keys { for i := 0; i < m.replicas; i++ { @@ -61,7 +61,7 @@ func (m *Map) Add(keys ...string) { sort.Ints(m.keys) } -// Gets the closest item in the hash to the provided key. +// Get gets the closest item in the hash to the provided key. func (m *Map) Get(key string) string { if m.IsEmpty() { return "" diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..e5c26dca --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/golang/groupcache + +go 1.20 + +require github.com/golang/protobuf v1.5.4 + +require google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..bc2601b8 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/groupcache.go b/groupcache.go index 316ca494..bc123f1d 100644 --- a/groupcache.go +++ b/groupcache.go @@ -25,6 +25,7 @@ limitations under the License. package groupcache import ( + "context" "errors" "math/rand" "strconv" @@ -44,13 +45,13 @@ type Getter interface { // uniquely describe the loaded data, without an implicit // current time, and without relying on cache expiration // mechanisms. - Get(ctx Context, key string, dest Sink) error + Get(ctx context.Context, key string, dest Sink) error } // A GetterFunc implements Getter with a function. -type GetterFunc func(ctx Context, key string, dest Sink) error +type GetterFunc func(ctx context.Context, key string, dest Sink) error -func (f GetterFunc) Get(ctx Context, key string, dest Sink) error { +func (f GetterFunc) Get(ctx context.Context, key string, dest Sink) error { return f(ctx, key, dest) } @@ -170,6 +171,10 @@ type Group struct { // Stats are statistics on the group. Stats Stats + + // rand is only non-nil when testing, + // to get predictable results in TestPeers. + rand *rand.Rand } // flightGroup is defined as an interface which flightgroup.Group @@ -204,7 +209,7 @@ func (g *Group) initPeers() { } } -func (g *Group) Get(ctx Context, key string, dest Sink) error { +func (g *Group) Get(ctx context.Context, key string, dest Sink) error { g.peersOnce.Do(g.initPeers) g.Stats.Gets.Add(1) if dest == nil { @@ -233,7 +238,7 @@ func (g *Group) Get(ctx Context, key string, dest Sink) error { } // load loads key either by invoking the getter locally or by sending it to another machine. -func (g *Group) load(ctx Context, key string, dest Sink) (value ByteView, destPopulated bool, err error) { +func (g *Group) load(ctx context.Context, key string, dest Sink) (value ByteView, destPopulated bool, err error) { g.Stats.Loads.Add(1) viewi, err := g.loadGroup.Do(key, func() (interface{}, error) { // Check the cache again because singleflight can only dedup calls @@ -245,7 +250,7 @@ func (g *Group) load(ctx Context, key string, dest Sink) (value ByteView, destPo // be only one entry for this key. // // Consider the following serialized event ordering for two - // goroutines in which this callback gets called twice for hte + // goroutines in which this callback gets called twice for the // same key: // 1: Get("key") // 2: Get("key") @@ -292,7 +297,7 @@ func (g *Group) load(ctx Context, key string, dest Sink) (value ByteView, destPo return } -func (g *Group) getLocally(ctx Context, key string, dest Sink) (ByteView, error) { +func (g *Group) getLocally(ctx context.Context, key string, dest Sink) (ByteView, error) { err := g.getter.Get(ctx, key, dest) if err != nil { return ByteView{}, err @@ -300,7 +305,7 @@ func (g *Group) getLocally(ctx Context, key string, dest Sink) (ByteView, error) return dest.view() } -func (g *Group) getFromPeer(ctx Context, peer ProtoGetter, key string) (ByteView, error) { +func (g *Group) getFromPeer(ctx context.Context, peer ProtoGetter, key string) (ByteView, error) { req := &pb.GetRequest{ Group: &g.name, Key: &key, @@ -314,7 +319,13 @@ func (g *Group) getFromPeer(ctx Context, peer ProtoGetter, key string) (ByteView // TODO(bradfitz): use res.MinuteQps or something smart to // conditionally populate hotCache. For now just do it some // percentage of the time. - if rand.Intn(10) == 0 { + var pop bool + if g.rand != nil { + pop = g.rand.Intn(10) == 0 + } else { + pop = rand.Intn(10) == 0 + } + if pop { g.populateCache(key, value, &g.hotCache) } return value, nil diff --git a/groupcache_test.go b/groupcache_test.go index ea05cac4..1bfe278c 100644 --- a/groupcache_test.go +++ b/groupcache_test.go @@ -19,6 +19,7 @@ limitations under the License. package groupcache import ( + "context" "errors" "fmt" "hash/crc32" @@ -41,7 +42,7 @@ var ( stringc = make(chan string) - dummyCtx Context + dummyCtx = context.TODO() // cacheFills is the number of times stringGroup or // protoGroup's Getter have been called. Read using the @@ -58,7 +59,7 @@ const ( ) func testSetup() { - stringGroup = NewGroup(stringGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error { + stringGroup = NewGroup(stringGroupName, cacheSize, GetterFunc(func(_ context.Context, key string, dest Sink) error { if key == fromChan { key = <-stringc } @@ -66,7 +67,7 @@ func testSetup() { return dest.SetString("ECHO:" + key) })) - protoGroup = NewGroup(protoGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error { + protoGroup = NewGroup(protoGroupName, cacheSize, GetterFunc(func(_ context.Context, key string, dest Sink) error { if key == fromChan { key = <-stringc } @@ -78,7 +79,7 @@ func testSetup() { })) } -// tests that a Getter's Get method is only called once with two +// TestGetDupSuppressString tests that a Getter's Get method is only called once with two // outstanding callers. This is the string variant. func TestGetDupSuppressString(t *testing.T) { once.Do(testSetup) @@ -120,7 +121,7 @@ func TestGetDupSuppressString(t *testing.T) { } } -// tests that a Getter's Get method is only called once with two +// TestGetDupSuppressProto tests that a Getter's Get method is only called once with two // outstanding callers. This is the proto variant. func TestGetDupSuppressProto(t *testing.T) { once.Do(testSetup) @@ -230,7 +231,7 @@ type fakePeer struct { fail bool } -func (p *fakePeer) Get(_ Context, in *pb.GetRequest, out *pb.GetResponse) error { +func (p *fakePeer) Get(_ context.Context, in *pb.GetRequest, out *pb.GetResponse) error { p.hits++ if p.fail { return errors.New("simulated error from peer") @@ -249,21 +250,21 @@ func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) { return p[n], p[n] != nil } -// tests that peers (virtual, in-process) are hit, and how much. +// TestPeers tests that peers (virtual, in-process) are hit, and how much. func TestPeers(t *testing.T) { once.Do(testSetup) - rand.Seed(123) peer0 := &fakePeer{} peer1 := &fakePeer{} peer2 := &fakePeer{} peerList := fakePeers([]ProtoGetter{peer0, peer1, peer2, nil}) const cacheSize = 0 // disabled localHits := 0 - getter := func(_ Context, key string, dest Sink) error { + getter := func(_ context.Context, key string, dest Sink) error { localHits++ return dest.SetString("got:" + key) } testGroup := newGroup("TestPeers-group", cacheSize, GetterFunc(getter), peerList) + testGroup.rand = rand.New(rand.NewSource(123)) run := func(name string, n int, wantSummary string) { // Reset counters localHits = 0 @@ -387,7 +388,7 @@ func (g *orderedFlightGroup) Do(key string, fn func() (interface{}, error)) (int func TestNoDedup(t *testing.T) { const testkey = "testkey" const testval = "testval" - g := newGroup("testgroup", 1024, GetterFunc(func(_ Context, key string, dest Sink) error { + g := newGroup("testgroup", 1024, GetterFunc(func(_ context.Context, key string, dest Sink) error { return dest.SetString(testval) }), nil) diff --git a/http.go b/http.go index f37467a7..e0d391a5 100644 --- a/http.go +++ b/http.go @@ -18,6 +18,7 @@ package groupcache import ( "bytes" + "context" "fmt" "io" "net/http" @@ -38,13 +39,13 @@ const defaultReplicas = 50 type HTTPPool struct { // Context optionally specifies a context for the server to use when it // receives a request. - // If nil, the server uses a nil Context. - Context func(*http.Request) Context + // If nil, the server uses the request's context + Context func(*http.Request) context.Context // Transport optionally specifies an http.RoundTripper for the client // to use when it makes a request. // If nil, the client uses http.DefaultTransport. - Transport func(Context) http.RoundTripper + Transport func(context.Context) http.RoundTripper // this peer's base URL, e.g. "https://example.net:8000" self string @@ -157,9 +158,11 @@ func (p *HTTPPool) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "no such group: "+groupName, http.StatusNotFound) return } - var ctx Context + var ctx context.Context if p.Context != nil { ctx = p.Context(r) + } else { + ctx = r.Context() } group.Stats.ServerRequests.Add(1) @@ -181,7 +184,7 @@ func (p *HTTPPool) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type httpGetter struct { - transport func(Context) http.RoundTripper + transport func(context.Context) http.RoundTripper baseURL string } @@ -189,7 +192,7 @@ var bufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } -func (h *httpGetter) Get(context Context, in *pb.GetRequest, out *pb.GetResponse) error { +func (h *httpGetter) Get(ctx context.Context, in *pb.GetRequest, out *pb.GetResponse) error { u := fmt.Sprintf( "%v%v/%v", h.baseURL, @@ -200,9 +203,10 @@ func (h *httpGetter) Get(context Context, in *pb.GetRequest, out *pb.GetResponse if err != nil { return err } + req = req.WithContext(ctx) tr := http.DefaultTransport if h.transport != nil { - tr = h.transport(context) + tr = h.transport(ctx) } res, err := tr.RoundTrip(req) if err != nil { diff --git a/http_test.go b/http_test.go index b42edd7f..132c1173 100644 --- a/http_test.go +++ b/http_test.go @@ -17,6 +17,7 @@ limitations under the License. package groupcache import ( + "context" "errors" "flag" "log" @@ -85,14 +86,14 @@ func TestHTTPPool(t *testing.T) { // Dummy getter function. Gets should go to children only. // The only time this process will handle a get is when the // children can't be contacted for some reason. - getter := GetterFunc(func(ctx Context, key string, dest Sink) error { + getter := GetterFunc(func(ctx context.Context, key string, dest Sink) error { return errors.New("parent getter called; something's wrong") }) g := NewGroup("httpPoolTest", 1<<20, getter) for _, key := range testKeys(nGets) { var value string - if err := g.Get(nil, key, StringSink(&value)); err != nil { + if err := g.Get(context.TODO(), key, StringSink(&value)); err != nil { t.Fatal(err) } if suffix := ":" + key; !strings.HasSuffix(value, suffix) { @@ -116,7 +117,7 @@ func beChildForTestHTTPPool() { p := NewHTTPPool("http://" + addrs[*peerIndex]) p.Set(addrToURL(addrs)...) - getter := GetterFunc(func(ctx Context, key string, dest Sink) error { + getter := GetterFunc(func(ctx context.Context, key string, dest Sink) error { dest.SetString(strconv.Itoa(*peerIndex) + ":" + key) return nil }) diff --git a/lru/lru.go b/lru/lru.go index 532cc45e..eac1c766 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -25,7 +25,7 @@ type Cache struct { // an item is evicted. Zero means no limit. MaxEntries int - // OnEvicted optionally specificies a callback function to be + // OnEvicted optionally specifies a callback function to be // executed when an entry is purged from the cache. OnEvicted func(key Key, value interface{}) diff --git a/lru/lru_test.go b/lru/lru_test.go index b7d9d8ac..a14f439e 100644 --- a/lru/lru_test.go +++ b/lru/lru_test.go @@ -40,7 +40,7 @@ var getTests = []struct { {"string_hit", "myKey", "myKey", true}, {"string_miss", "myKey", "nonsense", false}, {"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true}, - {"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false}, + {"simple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false}, {"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}}, complexStruct{1, simpleStruct{2, "three"}}, true}, } diff --git a/peers.go b/peers.go index 1625ff04..552c0aee 100644 --- a/peers.go +++ b/peers.go @@ -19,17 +19,17 @@ limitations under the License. package groupcache import ( + "context" + pb "github.com/golang/groupcache/groupcachepb" ) -// Context is an opaque value passed through calls to the -// ProtoGetter. It may be nil if your ProtoGetter implementation does -// not require a context. -type Context interface{} +// Context is an alias to context.Context for backwards compatibility purposes. +type Context = context.Context // ProtoGetter is the interface that must be implemented by a peer. type ProtoGetter interface { - Get(context Context, in *pb.GetRequest, out *pb.GetResponse) error + Get(ctx context.Context, in *pb.GetRequest, out *pb.GetResponse) error } // PeerPicker is the interface that must be implemented to locate diff --git a/singleflight/singleflight_test.go b/singleflight/singleflight_test.go index 47b4d3dc..f1b6e117 100644 --- a/singleflight/singleflight_test.go +++ b/singleflight/singleflight_test.go @@ -40,7 +40,7 @@ func TestDo(t *testing.T) { func TestDoErr(t *testing.T) { var g Group - someErr := errors.New("Some error") + someErr := errors.New("some error") v, err := g.Do("key", func() (interface{}, error) { return nil, someErr })