Token leases could outlive their TTL in HashiCorp Vault

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.23.0
-// 	protoc        v3.12.2
+// 	protoc        v3.13.0
 // source: helper/forwarding/types.proto
  
 package forwarding

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.23.0
-// 	protoc        v3.12.2
+// 	protoc        v3.13.0
 // source: helper/identity/mfa/types.proto
  
 package mfa

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.23.0
-// 	protoc        v3.12.2
+// 	protoc        v3.13.0
 // source: helper/identity/types.proto
  
 package identity

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.23.0
-// 	protoc        v3.12.2
+// 	protoc        v3.13.0
 // source: helper/storagepacker/types.proto
  
 package storagepacker

@@ -81,7 +81,7 @@ func TestRaft_Chunking_Lifecycle(t *testing.T) {
  
 	t.Log("tearing down cluster")
 	require.NoError(b.TeardownCluster(nil))
-	require.NoError(b.fsm.db.Close())
+	require.NoError(b.fsm.getDB().Close())
 	require.NoError(b.stableStore.(*raftboltdb.BoltStore).Close())
  
 	t.Log("starting new backend")
@@ -195,6 +195,15 @@ func TestRaft_Chunking_AppliedIndex(t *testing.T) {
 		t.Fatal(err)
 	}
  
+	// Write a value to fastforward the index
+	err = raft.Put(context.Background(), &physical.Entry{
+		Key:   "key",
+		Value: []byte("test"),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+ 
 	currentIndex := raft.AppliedIndex()
 	// Write some data
 	for i := 0; i < 10; i++ {

@@ -77,6 +77,9 @@ type FSM struct {
 	logger      log.Logger
 	noopRestore bool
  
+	// applyDelay is used to simulate a slow apply in tests
+	applyDelay time.Duration
+ 
 	db *bolt.DB
  
 	// retoreCb is called after we've restored a snapshot
@@ -118,6 +121,21 @@ func NewFSM(path string, logger log.Logger) (*FSM, error) {
 	return f, nil
 }
  
+func (f *FSM) getDB() *bolt.DB {
+	f.l.RLock()
+	defer f.l.RUnlock()
+ 
+	return f.db
+}
+ 
+// SetFSMDelay adds a delay to the FSM apply. This is used in tests to simulate
+// a slow apply.
+func (r *RaftBackend) SetFSMDelay(delay time.Duration) {
+	r.fsm.l.Lock()
+	r.fsm.applyDelay = delay
+	r.fsm.l.Unlock()
+}
+ 
 func (f *FSM) openDBFile(dbPath string) error {
 	if len(dbPath) == 0 {
 		return errors.New("can not open empty filename")
@@ -222,6 +240,9 @@ func writeSnapshotMetaToDB(metadata *raft.SnapshotMeta, db *bolt.DB) error {
 }
  
 func (f *FSM) witnessSnapshot(metadata *raft.SnapshotMeta) error {
+	f.l.RLock()
+	defer f.l.RUnlock()
+ 
 	err := writeSnapshotMetaToDB(metadata, f.db)
 	if err != nil {
 		return err
@@ -448,6 +469,10 @@ func (f *FSM) ApplyBatch(logs []*raft.Log) []interface{} {
 	f.l.RLock()
 	defer f.l.RUnlock()
  
+	if f.applyDelay > 0 {
+		time.Sleep(f.applyDelay)
+	}
+ 
 	err = f.db.Update(func(tx *bolt.Tx) error {
 		b := tx.Bucket(dataBucketName)
 		for _, commandRaw := range commands {

@@ -768,11 +768,14 @@ func (b *RaftBackend) AppliedIndex() uint64 {
 	b.l.RLock()
 	defer b.l.RUnlock()
  
-	if b.raft == nil {
+	if b.fsm == nil {
 		return 0
 	}
  
-	return b.raft.AppliedIndex()
+	// We use the latest index that the FSM has seen here, which may be behind
+	// raft.AppliedIndex() due to the async nature of the raft library.
+	indexState, _ := b.fsm.LatestState()
+	return indexState.Index
 }
  
 // RemovePeer removes the given peer ID from the raft cluster. If the node is

@@ -80,7 +80,7 @@ func getRaftWithDir(t testing.TB, bootstrap bool, noStoreState bool, raftDir str
 		}
  
 		for {
-			if backend.AppliedIndex() >= 2 {
+			if backend.raft.AppliedIndex() >= 2 {
 				break
 			}
 		}
@@ -156,7 +156,7 @@ func compareFSMsWithErr(t *testing.T, fsm1, fsm2 *FSM) error {
 		return fmt.Errorf("configs did not match: %+v != %+v", config1, config2)
 	}
  
-	return compareDBs(t, fsm1.db, fsm2.db, false)
+	return compareDBs(t, fsm1.getDB(), fsm2.getDB(), false)
 }
  
 func compareDBs(t *testing.T, boltDB1, boltDB2 *bolt.DB, dataOnly bool) error {