Remove namespace from RedisCheckpoint
The interface for checkpoint added another dependence to inject into the Pipeline. With this removed we can use the checkpoint directly from the pipeline.
This commit is contained in:
parent
da7a1dd87a
commit
71556ee7f7
4 changed files with 56 additions and 58 deletions
8
.travis.yml
Normal file
8
.travis.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
language: go
|
||||||
|
script: go test
|
||||||
|
notifications:
|
||||||
|
email: false
|
||||||
|
branches:
|
||||||
|
only:
|
||||||
|
- master
|
||||||
|
- development
|
||||||
|
|
@ -1,10 +1,48 @@
|
||||||
package connector
|
package connector
|
||||||
|
|
||||||
// Checkpoint is used by Pipeline.ProcessShard when they want to checkpoint their progress.
|
import (
|
||||||
// The Kinesis Connector Library will pass an object implementing this interface to ProcessShard,
|
"fmt"
|
||||||
// so they can checkpoint their progress.
|
|
||||||
type Checkpoint interface {
|
"github.com/hoisie/redis"
|
||||||
CheckpointExists(shardID string) bool
|
)
|
||||||
SequenceNumber() string
|
|
||||||
SetCheckpoint(shardID string, sequenceNumber string)
|
// Checkpoint implements the Checkpont interface.
|
||||||
|
// This class is used to enable the Pipeline.ProcessShard to checkpoint their progress.
|
||||||
|
type Checkpoint struct {
|
||||||
|
AppName string
|
||||||
|
StreamName string
|
||||||
|
|
||||||
|
client redis.Client
|
||||||
|
sequenceNumber string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckpointExists determines if a checkpoint for a particular Shard exists.
|
||||||
|
// Typically used to determine whether we should start processing the shard with
|
||||||
|
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
|
||||||
|
func (c *Checkpoint) CheckpointExists(shardID string) bool {
|
||||||
|
val, _ := c.client.Get(c.key(shardID))
|
||||||
|
|
||||||
|
if val != nil && string(val) != "" {
|
||||||
|
c.sequenceNumber = string(val)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// SequenceNumber returns the current checkpoint stored for the specified shard.
|
||||||
|
func (c *Checkpoint) SequenceNumber() string {
|
||||||
|
return c.sequenceNumber
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
|
||||||
|
// Upon failover, record processing is resumed from this point.
|
||||||
|
func (c *Checkpoint) SetCheckpoint(shardID string, sequenceNumber string) {
|
||||||
|
c.client.Set(c.key(shardID), []byte(sequenceNumber))
|
||||||
|
c.sequenceNumber = sequenceNumber
|
||||||
|
}
|
||||||
|
|
||||||
|
// key generates a unique Redis key for storage of Checkpoint.
|
||||||
|
func (c *Checkpoint) key(shardID string) string {
|
||||||
|
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ import (
|
||||||
|
|
||||||
func TestKey(t *testing.T) {
|
func TestKey(t *testing.T) {
|
||||||
k := "app:checkpoint:stream:shard"
|
k := "app:checkpoint:stream:shard"
|
||||||
c := RedisCheckpoint{AppName: "app", StreamName: "stream"}
|
c := Checkpoint{AppName: "app", StreamName: "stream"}
|
||||||
|
|
||||||
r := c.key("shard")
|
r := c.key("shard")
|
||||||
|
|
||||||
|
|
@ -21,7 +21,7 @@ func TestCheckpointExists(t *testing.T) {
|
||||||
var rc redis.Client
|
var rc redis.Client
|
||||||
k := "app:checkpoint:stream:shard"
|
k := "app:checkpoint:stream:shard"
|
||||||
rc.Set(k, []byte("fakeSeqNum"))
|
rc.Set(k, []byte("fakeSeqNum"))
|
||||||
c := RedisCheckpoint{AppName: "app", StreamName: "stream"}
|
c := Checkpoint{AppName: "app", StreamName: "stream"}
|
||||||
|
|
||||||
r := c.CheckpointExists("shard")
|
r := c.CheckpointExists("shard")
|
||||||
|
|
||||||
|
|
@ -35,7 +35,7 @@ func TestCheckpointExists(t *testing.T) {
|
||||||
func TestSetCheckpoint(t *testing.T) {
|
func TestSetCheckpoint(t *testing.T) {
|
||||||
k := "app:checkpoint:stream:shard"
|
k := "app:checkpoint:stream:shard"
|
||||||
var rc redis.Client
|
var rc redis.Client
|
||||||
c := RedisCheckpoint{AppName: "app", StreamName: "stream"}
|
c := Checkpoint{AppName: "app", StreamName: "stream"}
|
||||||
c.SetCheckpoint("shard", "fakeSeqNum")
|
c.SetCheckpoint("shard", "fakeSeqNum")
|
||||||
|
|
||||||
r, _ := rc.Get(k)
|
r, _ := rc.Get(k)
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
package connector
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/hoisie/redis"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RedisCheckpoint implements the Checkpont interface.
|
|
||||||
// This class is used to enable the Pipeline.ProcessShard to checkpoint their progress.
|
|
||||||
type RedisCheckpoint struct {
|
|
||||||
AppName string
|
|
||||||
StreamName string
|
|
||||||
|
|
||||||
client redis.Client
|
|
||||||
sequenceNumber string
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckpointExists determines if a checkpoint for a particular Shard exists.
|
|
||||||
// Typically used to determine whether we should start processing the shard with
|
|
||||||
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
|
|
||||||
func (c *RedisCheckpoint) CheckpointExists(shardID string) bool {
|
|
||||||
val, _ := c.client.Get(c.key(shardID))
|
|
||||||
|
|
||||||
if val != nil && string(val) != "" {
|
|
||||||
c.sequenceNumber = string(val)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// SequenceNumber returns the current checkpoint stored for the specified shard.
|
|
||||||
func (c *RedisCheckpoint) SequenceNumber() string {
|
|
||||||
return c.sequenceNumber
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
|
|
||||||
// Upon failover, record processing is resumed from this point.
|
|
||||||
func (c *RedisCheckpoint) SetCheckpoint(shardID string, sequenceNumber string) {
|
|
||||||
c.client.Set(c.key(shardID), []byte(sequenceNumber))
|
|
||||||
c.sequenceNumber = sequenceNumber
|
|
||||||
}
|
|
||||||
|
|
||||||
// key generates a unique Redis key for storage of Checkpoint.
|
|
||||||
func (c *RedisCheckpoint) key(shardID string) string {
|
|
||||||
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue