From 3757a433188b6f35a4229ca54ace1856f829c440 Mon Sep 17 00:00:00 2001 From: Dmitry Fedotov Date: Sun, 10 Aug 2025 13:02:38 +0300 Subject: [PATCH] feat: objects in vault are stored as a single secret Co-authored-by: Dmitry Fedotov Co-committed-by: Dmitry Fedotov --- .gitignore | 1 + internal/filesystem/fs_storage.go | 2 +- internal/vault/vault.go | 117 +++++++++++++++++++++--------- internal/vault/vault_test.go | 2 +- storage.go | 12 +-- 5 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..275179e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +internal/vault/test.sh diff --git a/internal/filesystem/fs_storage.go b/internal/filesystem/fs_storage.go index 20050be..a81c030 100644 --- a/internal/filesystem/fs_storage.go +++ b/internal/filesystem/fs_storage.go @@ -117,7 +117,7 @@ func validateKey(key string) error { func getPrefixPath(key string) string { r := []rune(key) - out := []rune{r[0], '/', r[1]} + out := []rune{r[0], os.PathSeparator, r[1]} return string(out) } diff --git a/internal/vault/vault.go b/internal/vault/vault.go index c2dd60d..3d7df72 100644 --- a/internal/vault/vault.go +++ b/internal/vault/vault.go @@ -4,9 +4,13 @@ import ( "context" "encoding/base64" "errors" + "fmt" + "path" + "strings" + + "github.com/hashicorp/vault/api" "code.uint32.ru/tiny/storage/internal/errinternal" - "github.com/hashicorp/vault/api" ) var ( @@ -14,25 +18,33 @@ var ( ) type Storage struct { - kv *api.KVv1 + key string + kv *api.KVv1 // TODO: kv2: *api.KVv2 } // New returns Storage writing to the specified vault path. -// Object will be base64 encoded and written to path/key. -func New(c *api.Client, path string) *Storage { - return &Storage{kv: c.KVv1(path)} +// Objects will be base64 encoded and written to path as a single +// secret. +func New(c *api.Client, secretpath string) *Storage { + p, k := path.Split(secretpath) + p = strings.Trim(p, "/") + k = strings.Trim(k, "/") + + return &Storage{kv: c.KVv1(p), key: k} } func (s *Storage) Save(key string, data []byte) error { - str := base64.StdEncoding.EncodeToString(data) - m := map[string]any{ - "data": map[string]string{ - "payload": str, - }, + datamap, err := s.getData() + if err != nil { + return err } - if err := s.kv.Put(context.Background(), "testkey", m); err != nil { + str := base64.StdEncoding.EncodeToString(data) + + datamap[key] = str + + if err := s.putData(datamap); err != nil { return err } @@ -40,9 +52,54 @@ func (s *Storage) Save(key string, data []byte) error { } func (s *Storage) Load(key string) ([]byte, error) { - m, err := s.kv.Get(context.Background(), key) + data, err := s.getData() + if err != nil { + return nil, err + } + + payload, ok := data[key] + if !ok { + return nil, errors.Join(ErrNotFound, errors.New("key not found in stored secret")) + } + + str, ok := payload.(string) + if !ok { + // returning ErrNotFound because this will fail again on retry + return nil, errors.Join(ErrNotFound, errors.New("could not convert payload to string")) + } + + b := []byte{} + + b, err = base64.StdEncoding.AppendDecode(b, []byte(str)) + if err != nil { + return nil, errors.Join(ErrNotFound, fmt.Errorf("could not base64 decode value: %w", err)) + } + + return b, nil +} + +func (s *Storage) putData(data map[string]any) error { + m := map[string]any{ + "data": data, + } + + if err := s.kv.Put(context.Background(), s.key, m); err != nil { + return err + } + + return nil +} + +func (s *Storage) getData() (map[string]any, error) { + m, err := s.kv.Get(context.Background(), s.key) if err != nil && errors.Is(err, api.ErrSecretNotFound) { - return nil, errors.Join(ErrNotFound, err) + // will create secret with no payload + m := make(map[string]any) + if err := s.putData(m); err != nil { + return nil, err + } + + return m, nil } else if err != nil { return nil, err } @@ -52,33 +109,27 @@ func (s *Storage) Load(key string) ([]byte, error) { return nil, errors.New("no data found") } - payloadmap, ok := data.(map[string]any) + datamap, ok := data.(map[string]any) if !ok { return nil, errors.New("no payload map") } - rawb, ok := payloadmap["payload"] - if !ok { - return nil, errors.New("no payload bytes") - } - - str, ok := rawb.(string) - if !ok { - return nil, errors.New("could not convert payload to bytes") - } - - b := []byte{} - - b, err = base64.StdEncoding.AppendDecode(b, []byte(str)) - if err != nil { - return nil, err - } - - return b, nil + return datamap, nil } func (s *Storage) Delete(key string) error { - if err := s.kv.Delete(context.Background(), key); err != nil { + data, err := s.getData() + if err != nil { + return err + } + + if _, ok := data[key]; !ok { + return nil + } + + delete(data, key) + + if err := s.putData(data); err != nil { return err } diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go index 7a669b7..dc4227d 100644 --- a/internal/vault/vault_test.go +++ b/internal/vault/vault_test.go @@ -38,7 +38,7 @@ func TestVaultStorage(t *testing.T) { data := []byte("this is a test") if err := st.Save(testkey, data); err != nil { - t.Error(err) + t.Fatalf("error saving data: %v", err) } b, err := st.Load(testkey) diff --git a/storage.go b/storage.go index 9762251..2192de9 100644 --- a/storage.go +++ b/storage.go @@ -53,11 +53,13 @@ func NewFS(path string) (Storage, error) { } // NewVault uses provided Vault client to store objects. -// The provided path is used as base path for -// keys. Objects saved to Storage will be put at -// /path/key as new secrets. -// Bytes passed to storage will be base64 encoded and saved -// in Vault as string. +// All objects are stored as a single secret, a JSON object +// where key are keys and values are base64 encoded bytes of +// saved object. +// If secret specified by path does not exist it will be created +// on first call to Storage methods. +// Note that a secret in vault gets updated (a new version of secret is created) +// on every save/delete operation. func NewVault(client *api.Client, path string) Storage { return vault.New(client, path) }