add kvstore (#1004)

* add kvstore

* use a fork of substrate rpc client

* fix lint

* use client decode function

* move decodeSecondKey to utils.go

Author: Eslam-Nawara

clients/tfchain-client-go/account.go

@@ -39,7 +39,7 @@ type AccountID types.AccountID
 // Balance
 type Balance struct {
 	Free       types.U128 `json:"free"`
-	Reserved   types.U128 `json:"reserverd"`
+	Reserved   types.U128 `json:"reserved"`
 	MiscFrozen types.U128 `json:"misc_frozen"`
 	FreeFrozen types.U128 `json:"free_frozen"`
 }
@@ -54,7 +54,7 @@ type AccountInfo struct {
 
 // TODO: Add service response status code if avialble
 type ActivationServiceError struct {
-	StatusCode int // 0 or a valid HTTP status code. 0 indicates that a response was not recived due to an error such host unreachable.
+	StatusCode int // 0 or a valid HTTP status code. 0 indicates that a response was not received due to an error such host unreachable.
 	Err        error
 }
 

clients/tfchain-client-go/go.mod

@@ -1,16 +1,16 @@
 module github.com/threefoldtech/tfchain/clients/tfchain-client-go
 
-go 1.17
+go 1.21
 
 require (
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.12
 	github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.26.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.2
 	github.com/vedhavyas/go-subkey v1.0.3
-	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871
+	golang.org/x/crypto v0.7.0
 )
 
 require (
@@ -19,20 +19,24 @@ require (
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/decred/base58 v1.0.3 // indirect
+	github.com/decred/base58 v1.0.4 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
-	github.com/ethereum/go-ethereum v1.10.17 // indirect
+	github.com/ethereum/go-ethereum v1.10.20 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/gtank/ristretto255 v0.1.2 // indirect
-	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
 	github.com/pierrec/xxHash v0.1.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
+	github.com/vedhavyas/go-subkey/v2 v2.0.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/centrifuge/go-substrate-rpc-client/v4 => github.com/threefoldtech/go-substrate-rpc-client/v4 v4.0.0-20240916115924-b6e9bfa88b8a

clients/tfchain-client-go/go.sum

@@ -0,0 +1,133 @@
+package substrate
+
+import (
+	"github.com/centrifuge/go-substrate-rpc-client/v4/types"
+
+	"github.com/pkg/errors"
+)
+
+func (s *Substrate) KVStoreSet(identity Identity, key string, value string) error {
+	cl, meta, err := s.GetClient()
+	if err != nil {
+		return err
+	}
+
+	c, err := types.NewCall(meta, "TFKVStore.set",
+		key, value,
+	)
+	if err != nil {
+		return errors.Wrap(err, "failed to create call")
+	}
+
+	res, err := s.Call(cl, meta, identity, c)
+	if err != nil {
+		return errors.Wrap(err, "failed to create contract")
+	}
+
+	if err := s.checkForError(res); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Substrate) KVStoreDelete(identity Identity, key string) error {
+	cl, meta, err := s.GetClient()
+	if err != nil {
+		return err
+	}
+
+	c, err := types.NewCall(meta, "TFKVStore.delete",
+		key,
+	)
+	if err != nil {
+		return errors.Wrap(err, "failed to create call")
+	}
+
+	res, err := s.Call(cl, meta, identity, c)
+	if err != nil {
+		return errors.Wrap(err, "failed to create contract")
+	}
+
+	if err := s.checkForError(res); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *Substrate) KVStoreGet(identity Identity, key string) ([]byte, error) {
+	cl, meta, err := s.GetClient()
+	if err != nil {
+		return nil, err
+	}
+
+	bytes, err := Encode(key)
+	if err != nil {
+		return nil, err
+	}
+
+	storageKey, err := types.CreateStorageKey(meta, "TFKVStore", "TFKVStore", identity.PublicKey(), bytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create substrate query key")
+	}
+
+	var value []byte
+	ok, err := cl.RPC.State.GetStorageLatest(storageKey, &value)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to lookup entity")
+	}
+
+	if !ok {
+		return nil, errors.Wrap(ErrNotFound, "key not found")
+	}
+
+	return value, nil
+}
+
+type Val struct {
+	Id  string
+	Key string
+}
+
+func (s *Substrate) KVStoreList(identity Identity) (map[string]string, error) {
+	cl, meta, err := s.GetClient()
+	if err != nil {
+		return nil, err
+	}
+
+	storageKey, err := types.CreateStorageKey(meta, "TFKVStore", "TFKVStore", identity.PublicKey())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create substrate query key")
+	}
+
+	keys, err := cl.RPC.State.GetKeysLatest(storageKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to lookup entity")
+	}
+
+	query, err := cl.RPC.State.QueryStorageAtLatest(keys)
+	if err != nil {
+		return nil, err
+	}
+
+	pairs := make(map[string]string)
+	for _, q := range query {
+		for _, c := range q.Changes {
+			var key, val []byte
+
+			key, err = decodeSecondKey(c.StorageKey, identity)
+			if err != nil {
+				return nil, err
+			}
+
+			err = Decode(c.StorageData, &val)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to decode value %v", string(c.StorageData))
+			}
+
+			pairs[string(key)] = string(val)
+		}
+	}
+
+	return pairs, nil
+}

clients/tfchain-client-go/kvstore.go

@@ -0,0 +1,51 @@
+package substrate
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKVStore(t *testing.T) {
+	pairs := map[string]string{"key1": "value1", "key2": "value2", "key3": "value3"}
+
+	// Alice identity
+	id, err := NewIdentityFromSr25519Phrase("//Alice")
+	require.NoError(t, err)
+
+	sub := startLocalConnection(t)
+	defer sub.Close()
+
+	t.Run("kvstore set values", func(t *testing.T) {
+		for key, val := range pairs {
+			err = sub.KVStoreSet(id, key, val)
+			assert.NoError(t, err)
+		}
+	})
+
+	t.Run("kvstore get values", func(t *testing.T) {
+		for key, val := range pairs {
+			value, err := sub.KVStoreGet(id, key)
+			assert.NoError(t, err)
+			assert.Equal(t, []byte(val), value)
+		}
+	})
+
+	t.Run("kvstore list keys", func(t *testing.T) {
+		values, err := sub.KVStoreList(id)
+		assert.NoError(t, err)
+		assert.EqualValues(t, values, pairs)
+	})
+
+	t.Run("kvstore delete", func(t *testing.T) {
+		for key := range pairs {
+			err = sub.KVStoreDelete(id, key)
+			assert.NoError(t, err)
+		}
+
+		values, err := sub.KVStoreList(id)
+		assert.NoError(t, err)
+		assert.Empty(t, values)
+	})
+}

clients/tfchain-client-go/kvstore_test.go

@@ -85,7 +85,7 @@ func TestServiceContract(t *testing.T) {
 	})
 
 	// 4. Bill consumer for service contract
-	// should be able to go to future block to test varaible amount greater than 0
+	// should be able to go to future block to test variable amount greater than 0
 
 	err = cl.ServiceContractBill(serviceIdentity, serviceContractID, variableAmount, billMetadata)
 	require.NoError(t, err)

clients/tfchain-client-go/service_contract_test.go

@@ -335,7 +335,6 @@ func (s *Substrate) sign(e *types.Extrinsic, signer Identity, o types.SignatureO
 	}
 
 	signerPubKey, err := types.NewMultiAddressFromAccountID(signer.PublicKey())
-
 	if err != nil {
 		return err
 	}
@@ -346,7 +345,6 @@ func (s *Substrate) sign(e *types.Extrinsic, signer Identity, o types.SignatureO
 	}
 
 	sig, err := signer.Sign(b)
-
 	if err != nil {
 		return err
 	}
@@ -412,7 +410,7 @@ func (s *Substrate) CallOnce(cl Conn, meta Meta, identity Identity, call types.C
 		return hash, err
 	}
 
-	//node.Address =identity.PublicKey
+	// node.Address =identity.PublicKey
 	account, err := s.getAccount(cl, meta, identity)
 	if err != nil {
 		return hash, errors.Wrap(err, "failed to get account")
@@ -536,15 +534,37 @@ func (s *Substrate) checkForError(callResponse *CallResponse) error {
 			if *accId == who {
 				if int(e.DispatchError.ModuleError.Index) < len(moduleErrors) {
 					if int(errIndex) >= len(moduleErrors[e.DispatchError.ModuleError.Index]) || moduleErrors[e.DispatchError.ModuleError.Index] == nil {
-						return fmt.Errorf("module error (%d) with unknown code %d occured, please update the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
+						return fmt.Errorf("module error (%d) with unknown code %d occurred, please update the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
 					}
 					return errors.New(moduleErrors[e.DispatchError.ModuleError.Index][errIndex])
 				} else {
-					return fmt.Errorf("unknown module error (%d) with code %d occured, please create the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
+					return fmt.Errorf("unknown module error (%d) with code %d occurred, please create the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
 				}
 			}
 		}
 	}
 
 	return nil
 }
+
+// decodeSecondKey extracts and decodes the second key(Vec<u8>) from the storage key.
+func decodeSecondKey(storageKey types.StorageKey, identity Identity) (key []byte, err error) {
+	// remove 16 bytes(32 in hex) pallet and map prefixes.
+	// pallet prefix (8 bytes): twox64(pallet_name)
+	// map prefix (8bytes) twox64(map_name)
+	prefixLen := 32
+
+	// the storage key contains two keys (AccountID and Vec<u8>)
+	// remove the length of the first key(AccountID)
+	// the hasher `Blake2_128Concat` includes a 16-byte hash followed by the AccountID
+	firstKeyLen := 32 + len(identity.PublicKey())
+
+	offset := prefixLen + firstKeyLen
+
+	if len(storageKey) < offset {
+		return nil, errors.New(fmt.Sprintf("failed to decode second key, storage key len should not be less than %d bytes", offset))
+	}
+
+	err = Decode(storageKey[offset:], &key)
+	return key, err
+}