From 48878e843344996136eb16317b3066a9adff3501 Mon Sep 17 00:00:00 2001 From: Dmitry Fedotov Date: Sun, 27 Jul 2025 19:02:05 +0300 Subject: [PATCH] feat: working version 1. implemented filesystem storage, NATS object storage and saving to Vault. 2. Test coverage is fine for filesystem and Vault (and NATS object does not really require extensive tests) --- LICENSE | 22 +++++++ README.md | 6 ++ go.mod | 18 +++--- go.sum | 74 +++++++++--------------- internal/errinternal/errors.go | 8 +++ internal/filesystem/fs_storage.go | 80 +++++++++++++++++++++----- internal/filesystem/fs_storage_test.go | 8 +-- internal/natsobj/nats_store.go | 52 ++++++----------- internal/vault/vault.go | 47 ++++++--------- internal/vault/vault_test.go | 6 +- storage.go | 43 +++++++++----- storageutil/nats.go | 36 ++++++++++++ storageutil/vault.go | 18 ++++++ 13 files changed, 265 insertions(+), 153 deletions(-) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 internal/errinternal/errors.go create mode 100644 storageutil/nats.go create mode 100644 storageutil/vault.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d6f303 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright 2025 Dmitry Fedotov + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the “Software”), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom +the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9a2c299 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# storage + +go get code.uint32.ru/tiny/storage + +This is a work in progress package. + diff --git a/go.mod b/go.mod index 585c608..4d94a88 100644 --- a/go.mod +++ b/go.mod @@ -2,21 +2,23 @@ module code.uint32.ru/tiny/storage go 1.24 -require github.com/nats-io/nats.go v1.43.0 +require ( + github.com/hashicorp/vault/api v1.20.0 + github.com/nats-io/nats.go v1.43.0 +) require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect - github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-7 // indirect - github.com/hashicorp/vault/api v1.20.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -24,8 +26,8 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/net v0.42.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect + golang.org/x/time v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index b9f9d0e..94c763e 100644 --- a/go.sum +++ b/go.sum @@ -1,85 +1,67 @@ -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= -github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= -github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault/api v1.20.0 h1:KQMHElgudOsr+IbJgmbjHnCTxEpKs9LnozA1D3nozU4= github.com/hashicorp/vault/api v1.20.0/go.mod h1:GZ4pcjfzoOWpkJ3ijHNpEoAxKEsBJnVljyTe3jM2Sms= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= -github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= -github.com/nats-io/nats.go v1.41.2 h1:5UkfLAtu/036s99AhFRlyNDI1Ieylb36qbGjJzHixos= -github.com/nats-io/nats.go v1.41.2/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug= github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= -github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/errinternal/errors.go b/internal/errinternal/errors.go new file mode 100644 index 0000000..03954b8 --- /dev/null +++ b/internal/errinternal/errors.go @@ -0,0 +1,8 @@ +package errinternal + +import "errors" + +var ( + ErrInvalidKey = errors.New("storage: invalid key") + ErrNotFound = errors.New("storage: not found") +) diff --git a/internal/filesystem/fs_storage.go b/internal/filesystem/fs_storage.go index 864a08a..20050be 100644 --- a/internal/filesystem/fs_storage.go +++ b/internal/filesystem/fs_storage.go @@ -1,12 +1,26 @@ package filesystem import ( + "errors" "fmt" "os" "path/filepath" + "strings" + + "code.uint32.ru/tiny/storage/internal/errinternal" ) -func Open(path string) (*Storage, error) { +const ( + fileModeDir os.FileMode = 0755 + fileModeFile os.FileMode = 0644 +) + +var ( + ErrInvalidKey = errinternal.ErrInvalidKey + ErrNotFound = errinternal.ErrNotFound +) + +func New(path string) (*Storage, error) { info, err := os.Stat(path) if err != nil { return nil, err @@ -21,17 +35,24 @@ func Open(path string) (*Storage, error) { return nil, fmt.Errorf("could not tarnslate %s to absolute path", path) } - return &Storage{prefix: abs}, nil + return &Storage{Dir: abs}, nil } type Storage struct { - prefix string + Dir string } func (s *Storage) Save(key string, data []byte) error { - path := s.toAbs(key) + if err := validateKey(key); err != nil { + return err + } - if err := os.WriteFile(path, data, 0664); err != nil { + path := s.getKeyPath(key) + if err := os.MkdirAll(path, fileModeDir); err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(path, key), data, fileModeFile); err != nil { return err } @@ -39,10 +60,16 @@ func (s *Storage) Save(key string, data []byte) error { } func (s *Storage) Load(key string) ([]byte, error) { - path := s.toAbs(key) + if err := validateKey(key); err != nil { + return nil, err + } - b, err := os.ReadFile(path) - if err != nil { + path := s.getKeyPath(key) + + b, err := os.ReadFile(filepath.Join(path, key)) + if err != nil && errors.Is(err, os.ErrNotExist) { + return nil, errors.Join(errinternal.ErrNotFound, err) + } else if err != nil { return nil, err } @@ -50,22 +77,47 @@ func (s *Storage) Load(key string) ([]byte, error) { } func (s *Storage) Delete(key string) error { - path := s.toAbs(key) + if err := validateKey(key); err != nil { + return err + } - err := os.Remove(path) - if err != nil && os.IsNotExist(err) { + path := s.getKeyPath(key) + + err := os.Remove(filepath.Join(path, key)) + if err != nil && errors.Is(err, os.ErrNotExist) { return nil } else if err != nil { return err } + // TODO: think of cleaning up path when no files left + // in /basedir/a/b/ after deleting key abc + return nil } -func (s *Storage) Close() error { +func (s *Storage) getKeyPath(key string) string { + return filepath.Join(s.Dir, getPrefixPath(key)) +} + +func validateKey(key string) error { + if len([]rune(key)) < 3 { + return errors.Join(ErrInvalidKey, fmt.Errorf("key must be at least 3 characters long")) + } + + // Of course windoze guys are missing the whole point, but + // let us use os-specific path separator and not ruin the whole fun + // for them :) + if strings.Contains(key, string(os.PathSeparator)) { + return errors.Join(ErrInvalidKey, fmt.Errorf("key must not contain path separator character: %s", string(os.PathSeparator))) + } + return nil } -func (s *Storage) toAbs(path string) string { - return filepath.Join(s.prefix, path) +func getPrefixPath(key string) string { + r := []rune(key) + out := []rune{r[0], '/', r[1]} + + return string(out) } diff --git a/internal/filesystem/fs_storage_test.go b/internal/filesystem/fs_storage_test.go index 9ea240d..bc55b70 100644 --- a/internal/filesystem/fs_storage_test.go +++ b/internal/filesystem/fs_storage_test.go @@ -7,15 +7,16 @@ import ( ) func TestStorageMethods(t *testing.T) { - st, err := Open("./testdata") + st, err := New("./testdata") if err != nil { t.Fatal(err) } name := "mytestfile" - data := []byte("contents of my test file") - defer os.Remove(name) // just in case + defer os.RemoveAll("./testdata/m") + + data := []byte("contents of my test file") if err := st.Save(name, data); err != nil { t.Fatal(err) @@ -41,5 +42,4 @@ func TestStorageMethods(t *testing.T) { if err := st.Delete(name); err != nil { t.Errorf("delete of non-existent failed: %v", err) } - } diff --git a/internal/natsobj/nats_store.go b/internal/natsobj/nats_store.go index 798ea74..ea21f91 100644 --- a/internal/natsobj/nats_store.go +++ b/internal/natsobj/nats_store.go @@ -1,41 +1,22 @@ package natsobj import ( + "errors" + + "code.uint32.ru/tiny/storage/internal/errinternal" "github.com/nats-io/nats.go" ) +var ( + ErrNotFound = errinternal.ErrNotFound +) + type Storage struct { store nats.ObjectStore - conn *nats.Conn } -func Open(bucket, url string) (*Storage, error) { - nc, err := nats.Connect(url) - if err != nil { - return nil, err - } - - js, err := nc.JetStream() - if err != nil { - return nil, err - } - - cfg := &nats.ObjectStoreConfig{ - Bucket: bucket, - Description: "tiny storage bucket", - MaxBytes: -1, - Storage: nats.FileStorage, - Compression: true, - } - - store, err := js.CreateObjectStore(cfg) - if err != nil { - return nil, err - } - - st := &Storage{store: store, conn: nc} - - return st, nil +func New(store nats.ObjectStore) *Storage { + return &Storage{store: store} } func (n *Storage) Save(key string, data []byte) error { @@ -47,7 +28,9 @@ func (n *Storage) Save(key string, data []byte) error { func (n *Storage) Load(key string) ([]byte, error) { b, err := n.store.GetBytes(key) - if err != nil { + if err != nil && errors.Is(err, nats.ErrObjectNotFound) { + return nil, errors.Join(ErrNotFound, err) + } else if err != nil { return nil, err } @@ -55,15 +38,12 @@ func (n *Storage) Load(key string) ([]byte, error) { } func (n *Storage) Delete(key string) error { - if err := n.store.Delete(key); err != nil { + err := n.store.Delete(key) + if err != nil && errors.Is(err, nats.ErrObjectNotFound) { + return nil + } else if err != nil { return err } return nil } - -func (n *Storage) Close() error { - n.conn.Close() - - return nil -} diff --git a/internal/vault/vault.go b/internal/vault/vault.go index 70364b7..c2dd60d 100644 --- a/internal/vault/vault.go +++ b/internal/vault/vault.go @@ -5,32 +5,26 @@ import ( "encoding/base64" "errors" + "code.uint32.ru/tiny/storage/internal/errinternal" "github.com/hashicorp/vault/api" ) +var ( + ErrNotFound = errinternal.ErrNotFound +) + type Storage struct { - client *api.Client - path string + kv *api.KVv1 + // TODO: kv2: *api.KVv2 } -func Open(token string, path string, addr string) (*Storage, error) { - conf := &api.Config{ - Address: addr, - } - - c, err := api.NewClient(conf) - if err != nil { - return nil, err - } - - c.SetToken(token) - - return &Storage{client: c, path: path}, nil +// New returns Storage writing to the specified vault path. +// Object will be base64 encoded and written to path/key. +func New(c *api.Client, path string) *Storage { + return &Storage{kv: c.KVv1(path)} } func (s *Storage) Save(key string, data []byte) error { - kv := s.client.KVv1(s.path) - str := base64.StdEncoding.EncodeToString(data) m := map[string]any{ "data": map[string]string{ @@ -38,7 +32,7 @@ func (s *Storage) Save(key string, data []byte) error { }, } - if err := kv.Put(context.Background(), "testkey", m); err != nil { + if err := s.kv.Put(context.Background(), "testkey", m); err != nil { return err } @@ -46,10 +40,10 @@ func (s *Storage) Save(key string, data []byte) error { } func (s *Storage) Load(key string) ([]byte, error) { - kv := s.client.KVv1(s.path) - - m, err := kv.Get(context.Background(), key) - if err != nil { + m, err := s.kv.Get(context.Background(), key) + if err != nil && errors.Is(err, api.ErrSecretNotFound) { + return nil, errors.Join(ErrNotFound, err) + } else if err != nil { return nil, err } @@ -84,16 +78,9 @@ func (s *Storage) Load(key string) ([]byte, error) { } func (s *Storage) Delete(key string) error { - kv := s.client.KVv1(s.path) - - if err := kv.Delete(context.Background(), key); err != nil { + if err := s.kv.Delete(context.Background(), key); err != nil { return err } return nil } - -func (s *Storage) Close() error { - s.client.ClearToken() - return nil -} diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go index 1893ded..7a669b7 100644 --- a/internal/vault/vault_test.go +++ b/internal/vault/vault_test.go @@ -4,6 +4,8 @@ import ( "bytes" "os" "testing" + + "code.uint32.ru/tiny/storage/storageutil" ) func TestVaultStorage(t *testing.T) { @@ -25,7 +27,9 @@ func TestVaultStorage(t *testing.T) { t.Log(addr) t.Log(path) - st, err := Open(token, path, addr) + client, err := storageutil.NewVaultApiClient(token, addr) + + st := New(client, path) if err != nil { t.Fatal(err) } diff --git a/storage.go b/storage.go index 1090b07..9762251 100644 --- a/storage.go +++ b/storage.go @@ -1,11 +1,24 @@ package storage import ( + "github.com/hashicorp/vault/api" + "github.com/nats-io/nats.go" + + "code.uint32.ru/tiny/storage/internal/errinternal" "code.uint32.ru/tiny/storage/internal/filesystem" "code.uint32.ru/tiny/storage/internal/natsobj" "code.uint32.ru/tiny/storage/internal/vault" ) +var ( + // ErrInvalidKey is returned when key validation + // for particular implementation of Storage fails. + ErrInvalidKey = errinternal.ErrInvalidKey + // ErrNotFound is returned when object is not found + // in Storage. + ErrNotFound = errinternal.ErrNotFound +) + var ( _ Storage = (*natsobj.Storage)(nil) _ Storage = (*filesystem.Storage)(nil) @@ -17,32 +30,34 @@ type Storage interface { // Save puts object with name 'key' into the store. // If a key already exists it gets overwritten. Save(key string, data []byte) error - // Load returns contents of object named 'key'. + // Load returns contents of object named 'key' or + // ErrNotFound. Load(key string) ([]byte, error) // Delete removes object named 'key' from the store. // If key does not exist Delete returns nil. Delete(key string) error - // Close must be called when you're done working with Storage. - Close() error } -// NewNats connects to NATS messaging system and tries to create -// a new object storage with name 'bucket'. The returned Storage -// uses the created bucket as underlying physical store. -func NewNats(bucket string, url string) (Storage, error) { - return natsobj.Open(bucket, url) +// NewNats wraps the provided ObjectStore with Storage interface. +func NewNats(store nats.ObjectStore) Storage { + return natsobj.New(store) } // NewFS established a key/value within the directory 'path' -// and uses is as underlying physical store. +// and uses it as underlying physical store. +// Note that the implementation requires keys to be at least +// 3 characters long. +// Key "abcd" will be stored in /path/a/b/abcd. func NewFS(path string) (Storage, error) { - return filesystem.Open(path) + return filesystem.New(path) } -// NewVault connects to Vault at addr and uses path as base path for +// NewVault uses provided Vault client to store objects. +// The provided path is used as base path for // keys. Objects saved to Storage will be put at // /path/key as new secrets. -// Bytes passed to storage will be base64 encoded and saved as string. -func NewVault(token string, path string, addr string) (Storage, error) { - return vault.Open(token, path, addr) +// Bytes passed to storage will be base64 encoded and saved +// in Vault as string. +func NewVault(client *api.Client, path string) Storage { + return vault.New(client, path) } diff --git a/storageutil/nats.go b/storageutil/nats.go new file mode 100644 index 0000000..a576a4b --- /dev/null +++ b/storageutil/nats.go @@ -0,0 +1,36 @@ +package storageutil + +import "github.com/nats-io/nats.go" + +// CreateNatsObjectStore is a convenience function that +// connects to NATS and using provided url and creates +// new object store using bucket as bucket name. +// The object store uses NATS file storage and compression. +// If fine-tuning is required - just create the store in your +// code and pass it to the storage package. +func CreateNatsObjectStore(url string, bucket string) (nats.ObjectStore, *nats.Conn, error) { + nc, err := nats.Connect(url) + if err != nil { + return nil, nil, err + } + + js, err := nc.JetStream() + if err != nil { + return nil, nil, err + } + + cfg := &nats.ObjectStoreConfig{ + Bucket: bucket, + Description: "tiny storage bucket", + MaxBytes: -1, + Storage: nats.FileStorage, + Compression: true, + } + + store, err := js.CreateObjectStore(cfg) + if err != nil { + return nil, nil, err + } + + return store, nc, nil +} diff --git a/storageutil/vault.go b/storageutil/vault.go new file mode 100644 index 0000000..03b99c0 --- /dev/null +++ b/storageutil/vault.go @@ -0,0 +1,18 @@ +package storageutil + +import "github.com/hashicorp/vault/api" + +func NewVaultApiClient(token string, addr string) (*api.Client, error) { + conf := &api.Config{ + Address: addr, + } + + c, err := api.NewClient(conf) + if err != nil { + return nil, err + } + + c.SetToken(token) + + return c, nil +}