From 735cfdbb1be0180677ccea2276fa93281051d7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zine=20Moualhi=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Fri, 15 Nov 2024 21:38:33 +0100 Subject: [PATCH] feat: added s3 proxy code --- apps/estrois/.env.example | 4 + apps/estrois/.gitignore | 6 +- apps/estrois/docker/compose.yml | 9 + apps/estrois/docker/garage/garage.toml | 21 ++ apps/estrois/go.mod | 25 +-- apps/estrois/go.sum | 51 ++--- apps/estrois/main.go | 270 +++++++++++++++++++------ 7 files changed, 265 insertions(+), 121 deletions(-) create mode 100644 apps/estrois/.env.example create mode 100644 apps/estrois/docker/compose.yml create mode 100644 apps/estrois/docker/garage/garage.toml diff --git a/apps/estrois/.env.example b/apps/estrois/.env.example new file mode 100644 index 0000000..00097ef --- /dev/null +++ b/apps/estrois/.env.example @@ -0,0 +1,4 @@ +export S3_ENDPOINT="your-s3-provider:9000" +export S3_ACCESS_KEY="your-access-key" +export S3_SECRET_KEY="your-secret-key" +export S3_USE_SSL="true" # if using HTTPS \ No newline at end of file diff --git a/apps/estrois/.gitignore b/apps/estrois/.gitignore index 0b96f24..72e4701 100644 --- a/apps/estrois/.gitignore +++ b/apps/estrois/.gitignore @@ -1,3 +1,7 @@ /target /cdn_root -/data \ No newline at end of file +/data +itsela-avatar.avif +docker/garage/* +!docker/garage/garage.toml +!docker/compose.yml \ No newline at end of file diff --git a/apps/estrois/docker/compose.yml b/apps/estrois/docker/compose.yml new file mode 100644 index 0000000..917f6ae --- /dev/null +++ b/apps/estrois/docker/compose.yml @@ -0,0 +1,9 @@ +services: + garage: + image: dxflrs/garage:v1.0.1 + network_mode: "host" + restart: unless-stopped + volumes: + - ./garage/garage.toml:/etc/garage.toml + - ./garage/meta:/var/lib/garage/meta + - ./garage/data:/var/lib/garage/data \ No newline at end of file diff --git a/apps/estrois/docker/garage/garage.toml b/apps/estrois/docker/garage/garage.toml new file mode 100644 index 0000000..4555100 --- /dev/null +++ b/apps/estrois/docker/garage/garage.toml @@ -0,0 +1,21 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" +metadata_auto_snapshot_interval = "6h" + +replication_factor = 1 + + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret = "af332d70a07d0891ff130b3bac1c6780231f9a7d870a0a5f7fef7985a0605b2a" + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" \ No newline at end of file diff --git a/apps/estrois/go.mod b/apps/estrois/go.mod index 1fc91ab..01aa574 100644 --- a/apps/estrois/go.mod +++ b/apps/estrois/go.mod @@ -3,45 +3,34 @@ module github.com/muandane/special-stack/estrois go 1.23.3 require ( - github.com/aws/aws-sdk-go-v2 v1.32.4 - github.com/aws/aws-sdk-go-v2/config v1.28.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 github.com/gin-gonic/gin v1.10.0 + github.com/minio/minio-go/v7 v7.0.80 ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.45 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.0 // indirect - github.com/aws/smithy-go v1.22.0 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.12.0 // indirect diff --git a/apps/estrois/go.sum b/apps/estrois/go.sum index 30651c7..2fd422a 100644 --- a/apps/estrois/go.sum +++ b/apps/estrois/go.sum @@ -1,39 +1,3 @@ -github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE= -github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.28.4 h1:qgD0MKmkIzZR2DrAjWJcI9UkndjR+8f6sjUQvXh0mb0= -github.com/aws/aws-sdk-go-v2/config v1.28.4/go.mod h1:LgnWnNzHZw4MLplSyEGia0WgJ/kCGD86zGCjvNpehJs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.45 h1:DUgm5lFso57E7150RBgu1JpVQoF8fAPretiDStIuVjg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.45/go.mod h1:dnBpENcPC1ekZrGpSWspX+ZRGzhkvqngT2Qp5xBR1dY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 h1:1SZBDiRzzs3sNhOMVApyWPduWYGAX0imGy06XiBnCAM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23/go.mod h1:i9TkxgbZmHVh2S0La6CAXtnyFhlCX/pJ0JsOvBAS6Mk= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 h1:aaPpoG15S2qHkWm4KlEyF01zovK1nW4BBbyXuHNSE90= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4/go.mod h1:eD9gS2EARTKgGr/W5xwgY/ik9z/zqpW+m/xOQbVxrMk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 h1:tHxQi/XHPK0ctd/wdOw0t7Xrc2OxcRCnVzv8lwWPu0c= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 h1:E5ZAVOmI2apR8ADb72Q63KqwwwdW1XcMeXIlrZ1Psjg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4/go.mod h1:wezzqVUOVVdk+2Z/JzQT4NxAU0NbhRe5W8pIE72jsWI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 h1:SwaJ0w0MOp0pBTIKTamLVeTKD+iOWyNJRdJ2KCQRg6Q= -github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0/go.mod h1:TMhLIyRIyoGVlaEMAt+ITMbwskSTpcGsCPDq91/ihY0= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 h1:HJwZwRt2Z2Tdec+m+fPjvdmkq2s9Ra+VR0hjF7V2o40= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 h1:zcx9LiGWZ6i6pjdcoE9oXAB6mUdeyC36Ia/QEiIvYdg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.0 h1:s7LRgBqhwLaxcocnAniBJp7gaAB+4I4vHzqUqjH18yc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.0/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8= -github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= -github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -46,12 +10,16 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -65,8 +33,13 @@ github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PU github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= @@ -75,6 +48,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= +github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -84,6 +61,8 @@ github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNH github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/apps/estrois/main.go b/apps/estrois/main.go index e545d1d..88190b0 100644 --- a/apps/estrois/main.go +++ b/apps/estrois/main.go @@ -4,34 +4,59 @@ import ( "bytes" "context" "fmt" - "net" + "io" "net/http" "os" + "sync" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gin-gonic/gin" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) -var s3Client *s3.Client +var minioClient *minio.Client + +// CacheEntry represents a cached object with metadata +type CacheEntry struct { + Data []byte + ContentType string + Size int64 + LastModified time.Time + ETag string + ExpiresAt time.Time +} + +// Cache configuration +const ( + defaultCacheDuration = 5 * time.Minute + maxCacheSize = 100 * 1024 * 1024 // 100MB + cleanupInterval = 1 * time.Minute +) + +var ( + cache sync.Map + cacheSize int64 + cacheMux sync.Mutex +) func init() { - region := getEnv("AWS_REGION", "us-east-1") - - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(region), - config.WithHTTPClient(&http.Client{ - Timeout: 30 * time.Second, - Transport: getS3TransportWithSigV4(), - }), - ) + endpoint := getEnv("S3_ENDPOINT", "localhost:9000") + accessKeyID := getEnv("S3_ACCESS_KEY", "minioadmin") + secretAccessKey := getEnv("S3_SECRET_KEY", "minioadmin") + useSSL := getEnv("S3_USE_SSL", "false") == "true" + + var err error + minioClient, err = minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { - panic(fmt.Sprintf("Failed to load AWS config: %v", err)) + panic(fmt.Sprintf("Failed to initialize Minio client: %v", err)) } - s3Client = s3.NewFromConfig(cfg) + // Start cache cleanup goroutine + go cleanupCache() } func getEnv(key, defaultValue string) string { @@ -39,10 +64,66 @@ func getEnv(key, defaultValue string) string { if value == "" { return defaultValue } - return value } +func cleanupCache() { + ticker := time.NewTicker(cleanupInterval) + for range ticker.C { + now := time.Now() + cache.Range(func(key, value interface{}) bool { + entry := value.(*CacheEntry) + if now.After(entry.ExpiresAt) { + cache.Delete(key) + cacheMux.Lock() + cacheSize -= entry.Size + cacheMux.Unlock() + } + return true + }) + } +} + +func getCacheKey(bucket, key string) string { + return fmt.Sprintf("%s/%s", bucket, key) +} + +func addToCache(cacheKey string, data []byte, contentType string, size int64, lastModified time.Time, etag string) { + // Check if adding this item would exceed the max cache size + cacheMux.Lock() + defer cacheMux.Unlock() + + if int64(len(data)) > maxCacheSize { + return // Don't cache files larger than max cache size + } + + // Remove old entries if necessary + if cacheSize+int64(len(data)) > maxCacheSize { + // Remove oldest entries until we have enough space + var keysToDelete []interface{} + cache.Range(func(key, value interface{}) bool { + keysToDelete = append(keysToDelete, key) + entry := value.(*CacheEntry) + cacheSize -= entry.Size + return cacheSize+int64(len(data)) > maxCacheSize + }) + for _, key := range keysToDelete { + cache.Delete(key) + } + } + + entry := &CacheEntry{ + Data: data, + ContentType: contentType, + Size: size, + LastModified: lastModified, + ETag: etag, + ExpiresAt: time.Now().Add(defaultCacheDuration), + } + cache.Store(cacheKey, entry) + cacheSize += size +} + func main() { r := gin.Default() r.GET("/objects/:bucket/*key", getObject) @@ -54,36 +135,85 @@ func main() { func getObject(c *gin.Context) { bucket := c.Param("bucket") - key := c.Param("key") + key := c.Param("key")[1:] + cacheKey := getCacheKey(bucket, key) - resp, err := s3Client.GetObject(context.TODO(), &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) + // Check cache first + if entry, ok := cache.Load(cacheKey); ok { + cacheEntry := entry.(*CacheEntry) + if time.Now().Before(cacheEntry.ExpiresAt) { + c.DataFromReader( + http.StatusOK, + cacheEntry.Size, + cacheEntry.ContentType, + io.NopCloser(io.NewSectionReader(bytes.NewReader(cacheEntry.Data), 0, cacheEntry.Size)), + nil, + ) + return + } + // Cache expired, remove it + cache.Delete(cacheKey) + cacheMux.Lock() + cacheSize -= cacheEntry.Size + cacheMux.Unlock() + } + + // Cache miss, get from S3 + obj, err := minioClient.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) + if err != nil { + c.AbortWithError(http.StatusInternalServerError, err) + return + } + + info, err := obj.Stat() + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + c.AbortWithStatus(http.StatusNotFound) + return + } + c.AbortWithError(http.StatusInternalServerError, err) + return + } + + // Read the entire object + data, err := io.ReadAll(obj) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } - defer resp.Body.Close() - c.DataFromReader(http.StatusOK, aws.ToInt64(resp.ContentLength), aws.ToString(resp.ContentType), resp.Body, nil) + // Cache the object + addToCache(cacheKey, data, info.ContentType, info.Size, info.LastModified, info.ETag) + + // Send response + c.DataFromReader( + http.StatusOK, + info.Size, + info.ContentType, + io.NopCloser(bytes.NewReader(data)), + nil, + ) } func putObject(c *gin.Context) { bucket := c.Param("bucket") - key := c.Param("key") + key := c.Param("key")[1:] + cacheKey := getCacheKey(bucket, key) - body, err := c.GetRawData() - if err != nil { - c.AbortWithError(http.StatusInternalServerError, err) - return + // Remove from cache if exists + if entry, ok := cache.LoadAndDelete(cacheKey); ok { + cacheMux.Lock() + cacheSize -= entry.(*CacheEntry).Size + cacheMux.Unlock() } - _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - Body: bytes.NewReader(body), - }) + contentType := c.GetHeader("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + _, err := minioClient.PutObject(context.Background(), bucket, key, c.Request.Body, -1, + minio.PutObjectOptions{ContentType: contentType}) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -94,12 +224,17 @@ func putObject(c *gin.Context) { func deleteObject(c *gin.Context) { bucket := c.Param("bucket") - key := c.Param("key") + key := c.Param("key")[1:] + cacheKey := getCacheKey(bucket, key) - _, err := s3Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) + // Remove from cache if exists + if entry, ok := cache.LoadAndDelete(cacheKey); ok { + cacheMux.Lock() + cacheSize -= entry.(*CacheEntry).Size + cacheMux.Unlock() + } + + err := minioClient.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{}) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -110,38 +245,41 @@ func deleteObject(c *gin.Context) { func headObject(c *gin.Context) { bucket := c.Param("bucket") - key := c.Param("key") + key := c.Param("key")[1:] + cacheKey := getCacheKey(bucket, key) - resp, err := s3Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) + // Check cache first + if entry, ok := cache.Load(cacheKey); ok { + cacheEntry := entry.(*CacheEntry) + if time.Now().Before(cacheEntry.ExpiresAt) { + c.Header("Content-Type", cacheEntry.ContentType) + c.Header("Content-Length", fmt.Sprintf("%d", cacheEntry.Size)) + c.Header("Last-Modified", cacheEntry.LastModified.UTC().Format(http.TimeFormat)) + c.Header("ETag", cacheEntry.ETag) + c.Status(http.StatusOK) + return + } + // Cache expired, remove it + cache.Delete(cacheKey) + cacheMux.Lock() + cacheSize -= cacheEntry.Size + cacheMux.Unlock() + } + + // Cache miss, get from S3 + info, err := minioClient.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{}) if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + c.AbortWithStatus(http.StatusNotFound) + return + } c.AbortWithError(http.StatusInternalServerError, err) return } - c.Header("Content-Type", aws.ToString(resp.ContentType)) - c.Header("Content-Length", fmt.Sprintf("%d", aws.ToInt64(resp.ContentLength))) + c.Header("Content-Type", info.ContentType) + c.Header("Content-Length", fmt.Sprintf("%d", info.Size)) + c.Header("Last-Modified", info.LastModified.UTC().Format(http.TimeFormat)) + c.Header("ETag", info.ETag) c.Status(http.StatusOK) } - -func getS3TransportWithSigV4() *http.Transport { - const timeout = 30 * time.Second - - dialer := &net.Dialer{ - Timeout: timeout, - KeepAlive: timeout, - DualStack: true, - } - - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: dialer.DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } -}