Remove namespace from RedisCheckpoint

The interface for checkpoint added another dependence to inject into the
Pipeline. With this removed we can use the checkpoint directly from the
pipeline.
This commit is contained in:
Harlow Ward 2014-12-21 01:08:44 -08:00
parent da7a1dd87a
commit 71556ee7f7
4 changed files with 56 additions and 58 deletions

8
.travis.yml Normal file
View file

@ -0,0 +1,8 @@
language: go
script: go test
notifications:
email: false
branches:
only:
- master
- development

View file

@ -1,10 +1,48 @@
package connector package connector
// Checkpoint is used by Pipeline.ProcessShard when they want to checkpoint their progress. import (
// The Kinesis Connector Library will pass an object implementing this interface to ProcessShard, "fmt"
// so they can checkpoint their progress.
type Checkpoint interface { "github.com/hoisie/redis"
CheckpointExists(shardID string) bool )
SequenceNumber() string
SetCheckpoint(shardID string, sequenceNumber string) // Checkpoint implements the Checkpont interface.
// This class is used to enable the Pipeline.ProcessShard to checkpoint their progress.
type Checkpoint struct {
AppName string
StreamName string
client redis.Client
sequenceNumber string
}
// CheckpointExists determines if a checkpoint for a particular Shard exists.
// Typically used to determine whether we should start processing the shard with
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
func (c *Checkpoint) CheckpointExists(shardID string) bool {
val, _ := c.client.Get(c.key(shardID))
if val != nil && string(val) != "" {
c.sequenceNumber = string(val)
return true
}
return false
}
// SequenceNumber returns the current checkpoint stored for the specified shard.
func (c *Checkpoint) SequenceNumber() string {
return c.sequenceNumber
}
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
// Upon failover, record processing is resumed from this point.
func (c *Checkpoint) SetCheckpoint(shardID string, sequenceNumber string) {
c.client.Set(c.key(shardID), []byte(sequenceNumber))
c.sequenceNumber = sequenceNumber
}
// key generates a unique Redis key for storage of Checkpoint.
func (c *Checkpoint) key(shardID string) string {
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
} }

View file

@ -8,7 +8,7 @@ import (
func TestKey(t *testing.T) { func TestKey(t *testing.T) {
k := "app:checkpoint:stream:shard" k := "app:checkpoint:stream:shard"
c := RedisCheckpoint{AppName: "app", StreamName: "stream"} c := Checkpoint{AppName: "app", StreamName: "stream"}
r := c.key("shard") r := c.key("shard")
@ -21,7 +21,7 @@ func TestCheckpointExists(t *testing.T) {
var rc redis.Client var rc redis.Client
k := "app:checkpoint:stream:shard" k := "app:checkpoint:stream:shard"
rc.Set(k, []byte("fakeSeqNum")) rc.Set(k, []byte("fakeSeqNum"))
c := RedisCheckpoint{AppName: "app", StreamName: "stream"} c := Checkpoint{AppName: "app", StreamName: "stream"}
r := c.CheckpointExists("shard") r := c.CheckpointExists("shard")
@ -35,7 +35,7 @@ func TestCheckpointExists(t *testing.T) {
func TestSetCheckpoint(t *testing.T) { func TestSetCheckpoint(t *testing.T) {
k := "app:checkpoint:stream:shard" k := "app:checkpoint:stream:shard"
var rc redis.Client var rc redis.Client
c := RedisCheckpoint{AppName: "app", StreamName: "stream"} c := Checkpoint{AppName: "app", StreamName: "stream"}
c.SetCheckpoint("shard", "fakeSeqNum") c.SetCheckpoint("shard", "fakeSeqNum")
r, _ := rc.Get(k) r, _ := rc.Get(k)

View file

@ -1,48 +0,0 @@
package connector
import (
"fmt"
"github.com/hoisie/redis"
)
// RedisCheckpoint implements the Checkpont interface.
// This class is used to enable the Pipeline.ProcessShard to checkpoint their progress.
type RedisCheckpoint struct {
AppName string
StreamName string
client redis.Client
sequenceNumber string
}
// CheckpointExists determines if a checkpoint for a particular Shard exists.
// Typically used to determine whether we should start processing the shard with
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
func (c *RedisCheckpoint) CheckpointExists(shardID string) bool {
val, _ := c.client.Get(c.key(shardID))
if val != nil && string(val) != "" {
c.sequenceNumber = string(val)
return true
}
return false
}
// SequenceNumber returns the current checkpoint stored for the specified shard.
func (c *RedisCheckpoint) SequenceNumber() string {
return c.sequenceNumber
}
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
// Upon failover, record processing is resumed from this point.
func (c *RedisCheckpoint) SetCheckpoint(shardID string, sequenceNumber string) {
c.client.Set(c.key(shardID), []byte(sequenceNumber))
c.sequenceNumber = sequenceNumber
}
// key generates a unique Redis key for storage of Checkpoint.
func (c *RedisCheckpoint) key(shardID string) string {
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
}