feat: working version

1. implemented filesystem storage, NATS object storage
and saving to Vault.
2. Test coverage is fine for filesystem and Vault
(and NATS object does not really require extensive tests)
This commit is contained in:
2025-07-27 19:02:05 +03:00
parent 854de3865b
commit 48878e8433
13 changed files with 265 additions and 153 deletions

22
LICENSE Normal file
View File

@@ -0,0 +1,22 @@
Copyright 2025 Dmitry Fedotov
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the “Software”), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom
the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall
be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

6
README.md Normal file
View File

@@ -0,0 +1,6 @@
# storage
go get code.uint32.ru/tiny/storage
This is a work in progress package.

18
go.mod
View File

@@ -2,21 +2,23 @@ module code.uint32.ru/tiny/storage
go 1.24
require github.com/nats-io/nats.go v1.43.0
require (
github.com/hashicorp/vault/api v1.20.0
github.com/nats-io/nats.go v1.43.0
)
require (
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
github.com/hashicorp/vault/api v1.20.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -24,8 +26,8 @@ require (
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
golang.org/x/crypto v0.40.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/net v0.42.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
golang.org/x/time v0.12.0 // indirect
)

74
go.sum
View File

@@ -1,85 +1,67 @@
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ=
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U=
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM=
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I=
github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
github.com/hashicorp/vault/api v1.20.0 h1:KQMHElgudOsr+IbJgmbjHnCTxEpKs9LnozA1D3nozU4=
github.com/hashicorp/vault/api v1.20.0/go.mod h1:GZ4pcjfzoOWpkJ3ijHNpEoAxKEsBJnVljyTe3jM2Sms=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nats.go v1.41.2 h1:5UkfLAtu/036s99AhFRlyNDI1Ieylb36qbGjJzHixos=
github.com/nats-io/nats.go v1.41.2/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,8 @@
package errinternal
import "errors"
var (
ErrInvalidKey = errors.New("storage: invalid key")
ErrNotFound = errors.New("storage: not found")
)

View File

@@ -1,12 +1,26 @@
package filesystem
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"code.uint32.ru/tiny/storage/internal/errinternal"
)
func Open(path string) (*Storage, error) {
const (
fileModeDir os.FileMode = 0755
fileModeFile os.FileMode = 0644
)
var (
ErrInvalidKey = errinternal.ErrInvalidKey
ErrNotFound = errinternal.ErrNotFound
)
func New(path string) (*Storage, error) {
info, err := os.Stat(path)
if err != nil {
return nil, err
@@ -21,17 +35,24 @@ func Open(path string) (*Storage, error) {
return nil, fmt.Errorf("could not tarnslate %s to absolute path", path)
}
return &Storage{prefix: abs}, nil
return &Storage{Dir: abs}, nil
}
type Storage struct {
prefix string
Dir string
}
func (s *Storage) Save(key string, data []byte) error {
path := s.toAbs(key)
if err := validateKey(key); err != nil {
return err
}
if err := os.WriteFile(path, data, 0664); err != nil {
path := s.getKeyPath(key)
if err := os.MkdirAll(path, fileModeDir); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(path, key), data, fileModeFile); err != nil {
return err
}
@@ -39,10 +60,16 @@ func (s *Storage) Save(key string, data []byte) error {
}
func (s *Storage) Load(key string) ([]byte, error) {
path := s.toAbs(key)
if err := validateKey(key); err != nil {
return nil, err
}
b, err := os.ReadFile(path)
if err != nil {
path := s.getKeyPath(key)
b, err := os.ReadFile(filepath.Join(path, key))
if err != nil && errors.Is(err, os.ErrNotExist) {
return nil, errors.Join(errinternal.ErrNotFound, err)
} else if err != nil {
return nil, err
}
@@ -50,22 +77,47 @@ func (s *Storage) Load(key string) ([]byte, error) {
}
func (s *Storage) Delete(key string) error {
path := s.toAbs(key)
if err := validateKey(key); err != nil {
return err
}
err := os.Remove(path)
if err != nil && os.IsNotExist(err) {
path := s.getKeyPath(key)
err := os.Remove(filepath.Join(path, key))
if err != nil && errors.Is(err, os.ErrNotExist) {
return nil
} else if err != nil {
return err
}
// TODO: think of cleaning up path when no files left
// in /basedir/a/b/ after deleting key abc
return nil
}
func (s *Storage) Close() error {
func (s *Storage) getKeyPath(key string) string {
return filepath.Join(s.Dir, getPrefixPath(key))
}
func validateKey(key string) error {
if len([]rune(key)) < 3 {
return errors.Join(ErrInvalidKey, fmt.Errorf("key must be at least 3 characters long"))
}
// Of course windoze guys are missing the whole point, but
// let us use os-specific path separator and not ruin the whole fun
// for them :)
if strings.Contains(key, string(os.PathSeparator)) {
return errors.Join(ErrInvalidKey, fmt.Errorf("key must not contain path separator character: %s", string(os.PathSeparator)))
}
return nil
}
func (s *Storage) toAbs(path string) string {
return filepath.Join(s.prefix, path)
func getPrefixPath(key string) string {
r := []rune(key)
out := []rune{r[0], '/', r[1]}
return string(out)
}

View File

@@ -7,15 +7,16 @@ import (
)
func TestStorageMethods(t *testing.T) {
st, err := Open("./testdata")
st, err := New("./testdata")
if err != nil {
t.Fatal(err)
}
name := "mytestfile"
data := []byte("contents of my test file")
defer os.Remove(name) // just in case
defer os.RemoveAll("./testdata/m")
data := []byte("contents of my test file")
if err := st.Save(name, data); err != nil {
t.Fatal(err)
@@ -41,5 +42,4 @@ func TestStorageMethods(t *testing.T) {
if err := st.Delete(name); err != nil {
t.Errorf("delete of non-existent failed: %v", err)
}
}

View File

@@ -1,41 +1,22 @@
package natsobj
import (
"errors"
"code.uint32.ru/tiny/storage/internal/errinternal"
"github.com/nats-io/nats.go"
)
var (
ErrNotFound = errinternal.ErrNotFound
)
type Storage struct {
store nats.ObjectStore
conn *nats.Conn
}
func Open(bucket, url string) (*Storage, error) {
nc, err := nats.Connect(url)
if err != nil {
return nil, err
}
js, err := nc.JetStream()
if err != nil {
return nil, err
}
cfg := &nats.ObjectStoreConfig{
Bucket: bucket,
Description: "tiny storage bucket",
MaxBytes: -1,
Storage: nats.FileStorage,
Compression: true,
}
store, err := js.CreateObjectStore(cfg)
if err != nil {
return nil, err
}
st := &Storage{store: store, conn: nc}
return st, nil
func New(store nats.ObjectStore) *Storage {
return &Storage{store: store}
}
func (n *Storage) Save(key string, data []byte) error {
@@ -47,7 +28,9 @@ func (n *Storage) Save(key string, data []byte) error {
func (n *Storage) Load(key string) ([]byte, error) {
b, err := n.store.GetBytes(key)
if err != nil {
if err != nil && errors.Is(err, nats.ErrObjectNotFound) {
return nil, errors.Join(ErrNotFound, err)
} else if err != nil {
return nil, err
}
@@ -55,15 +38,12 @@ func (n *Storage) Load(key string) ([]byte, error) {
}
func (n *Storage) Delete(key string) error {
if err := n.store.Delete(key); err != nil {
err := n.store.Delete(key)
if err != nil && errors.Is(err, nats.ErrObjectNotFound) {
return nil
} else if err != nil {
return err
}
return nil
}
func (n *Storage) Close() error {
n.conn.Close()
return nil
}

View File

@@ -5,32 +5,26 @@ import (
"encoding/base64"
"errors"
"code.uint32.ru/tiny/storage/internal/errinternal"
"github.com/hashicorp/vault/api"
)
var (
ErrNotFound = errinternal.ErrNotFound
)
type Storage struct {
client *api.Client
path string
kv *api.KVv1
// TODO: kv2: *api.KVv2
}
func Open(token string, path string, addr string) (*Storage, error) {
conf := &api.Config{
Address: addr,
}
c, err := api.NewClient(conf)
if err != nil {
return nil, err
}
c.SetToken(token)
return &Storage{client: c, path: path}, nil
// New returns Storage writing to the specified vault path.
// Object will be base64 encoded and written to path/key.
func New(c *api.Client, path string) *Storage {
return &Storage{kv: c.KVv1(path)}
}
func (s *Storage) Save(key string, data []byte) error {
kv := s.client.KVv1(s.path)
str := base64.StdEncoding.EncodeToString(data)
m := map[string]any{
"data": map[string]string{
@@ -38,7 +32,7 @@ func (s *Storage) Save(key string, data []byte) error {
},
}
if err := kv.Put(context.Background(), "testkey", m); err != nil {
if err := s.kv.Put(context.Background(), "testkey", m); err != nil {
return err
}
@@ -46,10 +40,10 @@ func (s *Storage) Save(key string, data []byte) error {
}
func (s *Storage) Load(key string) ([]byte, error) {
kv := s.client.KVv1(s.path)
m, err := kv.Get(context.Background(), key)
if err != nil {
m, err := s.kv.Get(context.Background(), key)
if err != nil && errors.Is(err, api.ErrSecretNotFound) {
return nil, errors.Join(ErrNotFound, err)
} else if err != nil {
return nil, err
}
@@ -84,16 +78,9 @@ func (s *Storage) Load(key string) ([]byte, error) {
}
func (s *Storage) Delete(key string) error {
kv := s.client.KVv1(s.path)
if err := kv.Delete(context.Background(), key); err != nil {
if err := s.kv.Delete(context.Background(), key); err != nil {
return err
}
return nil
}
func (s *Storage) Close() error {
s.client.ClearToken()
return nil
}

View File

@@ -4,6 +4,8 @@ import (
"bytes"
"os"
"testing"
"code.uint32.ru/tiny/storage/storageutil"
)
func TestVaultStorage(t *testing.T) {
@@ -25,7 +27,9 @@ func TestVaultStorage(t *testing.T) {
t.Log(addr)
t.Log(path)
st, err := Open(token, path, addr)
client, err := storageutil.NewVaultApiClient(token, addr)
st := New(client, path)
if err != nil {
t.Fatal(err)
}

View File

@@ -1,11 +1,24 @@
package storage
import (
"github.com/hashicorp/vault/api"
"github.com/nats-io/nats.go"
"code.uint32.ru/tiny/storage/internal/errinternal"
"code.uint32.ru/tiny/storage/internal/filesystem"
"code.uint32.ru/tiny/storage/internal/natsobj"
"code.uint32.ru/tiny/storage/internal/vault"
)
var (
// ErrInvalidKey is returned when key validation
// for particular implementation of Storage fails.
ErrInvalidKey = errinternal.ErrInvalidKey
// ErrNotFound is returned when object is not found
// in Storage.
ErrNotFound = errinternal.ErrNotFound
)
var (
_ Storage = (*natsobj.Storage)(nil)
_ Storage = (*filesystem.Storage)(nil)
@@ -17,32 +30,34 @@ type Storage interface {
// Save puts object with name 'key' into the store.
// If a key already exists it gets overwritten.
Save(key string, data []byte) error
// Load returns contents of object named 'key'.
// Load returns contents of object named 'key' or
// ErrNotFound.
Load(key string) ([]byte, error)
// Delete removes object named 'key' from the store.
// If key does not exist Delete returns nil.
Delete(key string) error
// Close must be called when you're done working with Storage.
Close() error
}
// NewNats connects to NATS messaging system and tries to create
// a new object storage with name 'bucket'. The returned Storage
// uses the created bucket as underlying physical store.
func NewNats(bucket string, url string) (Storage, error) {
return natsobj.Open(bucket, url)
// NewNats wraps the provided ObjectStore with Storage interface.
func NewNats(store nats.ObjectStore) Storage {
return natsobj.New(store)
}
// NewFS established a key/value within the directory 'path'
// and uses is as underlying physical store.
// and uses it as underlying physical store.
// Note that the implementation requires keys to be at least
// 3 characters long.
// Key "abcd" will be stored in /path/a/b/abcd.
func NewFS(path string) (Storage, error) {
return filesystem.Open(path)
return filesystem.New(path)
}
// NewVault connects to Vault at addr and uses path as base path for
// NewVault uses provided Vault client to store objects.
// The provided path is used as base path for
// keys. Objects saved to Storage will be put at
// /path/key as new secrets.
// Bytes passed to storage will be base64 encoded and saved as string.
func NewVault(token string, path string, addr string) (Storage, error) {
return vault.Open(token, path, addr)
// Bytes passed to storage will be base64 encoded and saved
// in Vault as string.
func NewVault(client *api.Client, path string) Storage {
return vault.New(client, path)
}

36
storageutil/nats.go Normal file
View File

@@ -0,0 +1,36 @@
package storageutil
import "github.com/nats-io/nats.go"
// CreateNatsObjectStore is a convenience function that
// connects to NATS and using provided url and creates
// new object store using bucket as bucket name.
// The object store uses NATS file storage and compression.
// If fine-tuning is required - just create the store in your
// code and pass it to the storage package.
func CreateNatsObjectStore(url string, bucket string) (nats.ObjectStore, *nats.Conn, error) {
nc, err := nats.Connect(url)
if err != nil {
return nil, nil, err
}
js, err := nc.JetStream()
if err != nil {
return nil, nil, err
}
cfg := &nats.ObjectStoreConfig{
Bucket: bucket,
Description: "tiny storage bucket",
MaxBytes: -1,
Storage: nats.FileStorage,
Compression: true,
}
store, err := js.CreateObjectStore(cfg)
if err != nil {
return nil, nil, err
}
return store, nc, nil
}

18
storageutil/vault.go Normal file
View File

@@ -0,0 +1,18 @@
package storageutil
import "github.com/hashicorp/vault/api"
func NewVaultApiClient(token string, addr string) (*api.Client, error) {
conf := &api.Config{
Address: addr,
}
c, err := api.NewClient(conf)
if err != nil {
return nil, err
}
c.SetToken(token)
return c, nil
}