From 4bbef398e4a895fba689abd8f8594f8af1486ad3 Mon Sep 17 00:00:00 2001 From: bruce Date: Mon, 4 Nov 2024 03:18:11 +0000 Subject: [PATCH] feat(utils): add utils. feat(uzsdk): 1, merge some variables to Configuration. --- README.md | 1 - go.mod | 54 +++-- go.sum | 139 ++++++----- utils/chrome/chromedp.go | 88 +++++++ utils/convutil/reflect.go | 442 +++++++++++++++++++++++++++++++++ utils/convutil/sort.go | 95 ++++++++ utils/dbutil/format.go | 499 ++++++++++++++++++++++++++++++++++++++ utils/dsutil/array.go | 168 +++++++++++++ utils/fmtutil/format.go | 55 +++++ utils/fsutil/archive.go | 184 ++++++++++++++ utils/fsutil/crypto.go | 166 +++++++++++++ utils/fsutil/encoding.go | 64 +++++ utils/fsutil/file.go | 128 ++++++++++ utils/netutil/mime.go | 65 +++++ utils/osutil/exec.go | 43 ++++ utils/osutil/path.go | 107 ++++++++ utils/osutil/syscall.go | 39 +++ utils/strutil/encoding.go | 33 +++ utils/strutil/escape.go | 80 ++++++ utils/strutil/strings.go | 195 +++++++++++++++ utils/utils.sh | 111 +++++++++ uzsdk/application.go | 3 +- uzsdk/configuration.go | 29 ++- uzsdk/service.go | 3 +- version.go | 68 +----- 25 files changed, 2714 insertions(+), 145 deletions(-) create mode 100644 utils/chrome/chromedp.go create mode 100644 utils/convutil/reflect.go create mode 100644 utils/convutil/sort.go create mode 100644 utils/dbutil/format.go create mode 100644 utils/dsutil/array.go create mode 100644 utils/fmtutil/format.go create mode 100644 utils/fsutil/archive.go create mode 100644 utils/fsutil/crypto.go create mode 100644 utils/fsutil/encoding.go create mode 100644 utils/fsutil/file.go create mode 100644 utils/netutil/mime.go create mode 100644 utils/osutil/exec.go create mode 100644 utils/osutil/path.go create mode 100644 utils/osutil/syscall.go create mode 100644 utils/strutil/encoding.go create mode 100644 utils/strutil/escape.go create mode 100644 utils/strutil/strings.go create mode 100644 utils/utils.sh diff --git a/README.md b/README.md index 69fe883..f9c7852 100644 --- a/README.md +++ b/README.md @@ -49,5 +49,4 @@ func main() { $ go run example.go ``` - ## Testing diff --git a/go.mod b/go.mod index f2added..6b08bf3 100644 --- a/go.mod +++ b/go.mod @@ -1,35 +1,51 @@ module git.uzoombox.com/git/uzsdk -go 1.20 +go 1.23 + +toolchain go1.23.2 require ( - github.com/gin-gonic/gin v1.9.1 - github.com/jmoiron/sqlx v1.3.5 + github.com/chromedp/cdproto v0.0.0-20241030022559-23c28aebe8cb + github.com/chromedp/chromedp v0.11.1 + github.com/cstockton/go-conv v1.0.0 + github.com/fatih/structs v1.1.0 + github.com/gin-gonic/gin v1.10.0 + github.com/jmoiron/sqlx v1.4.0 + github.com/spaolacci/murmur3 v1.1.0 + golang.org/x/sys v0.26.0 ) require ( - github.com/bytedance/sonic v1.9.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/bytedance/sonic v1.12.3 // indirect + github.com/bytedance/sonic/loader v0.2.1 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/text v0.19.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index eb9fa41..6e5aeb8 100644 --- a/go.sum +++ b/go.sum @@ -1,57 +1,89 @@ -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= +github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= +github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/chromedp/cdproto v0.0.0-20241030022559-23c28aebe8cb h1:yBPpAakATGLWZsVgYRcU9FopbOqzoazzbFaStQ9DCMc= +github.com/chromedp/cdproto v0.0.0-20241030022559-23c28aebe8cb/go.mod h1:4XqMl3iIW08jtieURWL6Tt5924w21pxirC6th662XUM= +github.com/chromedp/chromedp v0.11.1 h1:Spca8egFqUlv+JDW+yIs+ijlHlJDPufgrfXPwtq6NMs= +github.com/chromedp/chromedp v0.11.1/go.mod h1:lr8dFRLKsdTTWb75C/Ttol2vnBKOSnt0BW8R9Xaupi8= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cstockton/go-conv v1.0.0 h1:zj/q/0MpQ/97XfiC9glWiohO8lhgR4TTnHYZifLTv6I= +github.com/cstockton/go-conv v1.0.0/go.mod h1:HuiHkkRgOA0IoBNPC7ysG7kNpjDYlgM7Kj62yQPxjy4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -60,34 +92,29 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/utils/chrome/chromedp.go b/utils/chrome/chromedp.go new file mode 100644 index 0000000..829bff3 --- /dev/null +++ b/utils/chrome/chromedp.go @@ -0,0 +1,88 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package chrome + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/chromedp" +) + +// ClickandWait click an element with selector 'sel' if it's exist, and wait for done if 'wait' is true. +func ClickandWait(ctx context.Context, sel string, options ...interface{}) error { + wait := true + for _, option := range options { + switch option.(type) { + case bool: + wait = option.(bool) + } + } + if err := TryWaiting(ctx, sel, time.Millisecond*10, 1); err == nil { + var data []byte + if err := chromedp.EvaluateAsDevTools(fmt.Sprintf(`document.querySelector('%v').click()`, sel), &data).Do(ctx); err != nil { + log.Println(err) + return err + } else { + if wait { + return chromedp.WaitNotPresent(sel, chromedp.ByQuery).Do(ctx) + } else { + return nil + } + } + } else { + return errors.New("element not found") + } +} + +// FillInput fill an element with selector 'sel' using 'value'. +func FillInput(ctx context.Context, sel, value string) error { + if err := chromedp.Focus(sel, chromedp.ByQuery).Do(ctx); err != nil { + return err + } + if err := chromedp.Clear(sel, chromedp.ByQuery).Do(ctx); err != nil { + return err + } + if err := chromedp.SendKeys(sel, value, chromedp.ByQuery).Do(ctx); err != nil { + return err + } + if false { + var val string + chromedp.Value(sel, &val, chromedp.ByQuery).Do(ctx) + } + + return nil +} + +// TryWaiting try to wait an element with selector 'sel' for exist. +func TryWaiting(ctx context.Context, sel string, options ...interface{}) error { + interval := time.Millisecond * 20 + count := 10 + for _, option := range options { + switch option.(type) { + case time.Duration: + interval = option.(time.Duration) + case int: + count = option.(int) + } + } + + var nodes []*cdp.Node + for i := 0; i < count; i++ { + if err := chromedp.Nodes(sel, &nodes, chromedp.AtLeast(0)).Do(ctx); err != nil { + log.Println(err) + return err + } else { + if len(nodes) > 0 { + return nil + } + } + chromedp.Sleep(interval).Do(ctx) + } + + return errors.New("element not found") +} diff --git a/utils/convutil/reflect.go b/utils/convutil/reflect.go new file mode 100644 index 0000000..c8bebbb --- /dev/null +++ b/utils/convutil/reflect.go @@ -0,0 +1,442 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package convutil + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "github.com/cstockton/go-conv" +) + +// PrintStruct print information of struct, v can be ptr or struct. +func PrintStruct(v interface{}) error { + vv := reflect.ValueOf(v) + if vv.Kind() == reflect.Ptr && !vv.IsNil() { + vv = vv.Elem() + } + if vv.Kind() != reflect.Struct { + return fmt.Errorf("v must be a valid pointer or struct object.") + } + + for i := 0; i < vv.NumField(); i++ { + f := vv.Field(i) + if f.Kind() == reflect.Ptr { + f = f.Elem() + } + fmt.Printf("%v: %v\n", f.Kind(), f) + } + + return nil +} + +// AssignStruct set struct dst's value with map[string]interface{} src, +// if force=true: assign value anytime, +// else: assign value when field does not exist or field value is zero. +func AssignStruct(dst interface{}, src interface{}, force bool) error { + vd := reflect.ValueOf(dst) + vs := reflect.ValueOf(src) + if vd.Kind() != reflect.Ptr || vd.IsNil() { + return fmt.Errorf("dst is not a valid pointer.") + } + if vs.Kind() != reflect.Ptr || vs.IsNil() { + return fmt.Errorf("src is not a valid pointer.") + } + + vd = vd.Elem() + vs = vs.Elem() + if vd.Kind() != reflect.Struct { + return fmt.Errorf("dst must be a struct.") + } + if vs.Kind() != reflect.Map { + return fmt.Errorf("src must be a map.") + } + + for _, k := range vs.MapKeys() { + key := k.String() + val := vs.MapIndex(k) + if f := vd.FieldByName(key); f.Kind() != reflect.Invalid { + assign := force + if !assign { + if f.Kind() == reflect.Interface || f.Kind() == reflect.Ptr { + fv := f.Elem() + if !fv.IsValid() || fv.IsZero() { + assign = true + } + } + } + if assign { + if err := convertAssign(f, val); err != nil { + return err + } + } + } else { + return fmt.Errorf("f[%s] not found.", key) + } + } + + return nil +} + +// CopyStruct make a copy of struct v, +// if notnil=true: only copy fileds that value is not nil, +// else: copy all fields even if value is nil. +// options will be [excludes,includes]: +// +// excludes []string: MUST NOT copy fields. +// includes []string: MUST copy fields that even it's value is nil. +func CopyStruct(v interface{}, notnil bool, options ...interface{}) interface{} { + // parse options. + var excludes []string + var includes []string + for _, option := range options { + switch option.(type) { + case []string: + excludes = option.([]string) + case map[string]interface{}: + for k, v := range option.(map[string]interface{}) { + switch v.(type) { + case []string: + switch k { + case "excludes": + excludes = v.([]string) + case "includes": + includes = v.([]string) + } + } + } + } + } + + var nullres struct{} + vv := reflect.ValueOf(v) + if vv.Kind() != reflect.Ptr || vv.IsNil() { + fmt.Println("v is not a valid pointer.") + return nullres + } + vv = vv.Elem() + if vv.Kind() != reflect.Struct { + fmt.Println("v must be a struct.") + return nullres + } + + fields := make([]reflect.StructField, 0) + values := make([]reflect.Value, 0) + vt := reflect.TypeOf(vv.Interface()) + for i := 0; i < vv.NumField(); i++ { + if value := vv.Field(i); value.Kind() != reflect.Invalid { + field := vt.Field(i) + need := true + must := false + for _, exclude := range excludes { + if exclude == field.Name { + need = false + break + } + } + for _, include := range includes { + if include == field.Name { + must = true + break + } + } + valid := value.IsValid() + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + valid = !value.IsNil() + } + if (must) || (need && (valid || !notnil)) { + if 'A' <= field.Name[0] && 'Z' >= field.Name[0] { // fix: StructOf does not allow unexported fields + fields = append(fields, field) + values = append(values, value) + } + } + } + } + vd := reflect.New(reflect.StructOf(fields)).Elem() + for i := 0; i < len(fields); i++ { + vd.Field(i).Set(values[i]) + } + + return vd.Interface() +} + +// BindString make a string from value, +// and returns the point of it for binding. +func BindString(value interface{}) *string { + val, _ := conv.String(value) + return &val +} + +func convertAssign(dst, src reflect.Value) error { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return fmt.Errorf("src is not a pointer.") + } + + switch dst.Interface().(type) { + case *bool: + if s, err := asBool(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *int: + if t, err := asInt(src); err == nil { + s := int(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *int8: + if t, err := asInt(src); err == nil { + s := int8(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *int16: + if t, err := asInt(src); err == nil { + s := int16(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *int32: + if t, err := asInt(src); err == nil { + s := int32(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *int64: + if s, err := asInt(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *uint: + if t, err := asInt(src); err == nil { + s := uint(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *uint8: + if t, err := asInt(src); err == nil { + s := uint8(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *uint16: + if t, err := asInt(src); err == nil { + s := uint16(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *uint32: + if t, err := asInt(src); err == nil { + s := uint32(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *uint64: + if t, err := asInt(src); err == nil { + s := uint64(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *float32: + if t, err := asFloat(src); err == nil { + s := float32(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *float64: + if s, err := asFloat(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *complex64: + if t, err := asComplex(src); err == nil { + s := complex64(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *complex128: + if s, err := asComplex(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *string: + if s, err := asString(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *[]byte: + if t, err := asString(src); err == nil { + s := []byte(t) + dst.Set(reflect.ValueOf(&s)) + return nil + } + case *time.Time: + if s, err := asTime(src); err == nil { + dst.Set(reflect.ValueOf(&s)) + return nil + } + } + + return fmt.Errorf("unsupport data type: %#v=>%#v", src, dst) +} + +func asBool(src reflect.Value) (bool, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return false, fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case bool: + return src.Elem().Bool(), nil + case int, int8, int16, int32, int64: + return src.Elem().Int() != 0, nil + case uint, uint8, uint16, uint32, uint64: + return src.Elem().Uint() != 0, nil + case string: + s := strings.ToUpper(strings.TrimSpace(src.Elem().String())) + return s == "TRUE" || s == "YES" || s == "Y", nil + case *bool: + s := *src.Interface().(*bool) + return s, nil + } + + return false, fmt.Errorf("src cannot be converted") +} + +func asInt(src reflect.Value) (int64, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return 0, fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case int, int8, int16, int32, int64: + return int64(src.Elem().Int()), nil + case uint, uint8, uint16, uint32, uint64: + return int64(src.Elem().Uint()), nil + case float32, float64: + return int64(src.Elem().Float()), nil + case complex64, complex128: + return int64(real(src.Elem().Complex())), nil + case string: + s, _ := strconv.ParseInt(src.Elem().String(), 0, 64) + return s, nil + case *int64: + s := *src.Interface().(*int64) + return s, nil + } + + return 0, fmt.Errorf("src cannot be converted") +} + +func asFloat(src reflect.Value) (float64, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return 0, fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case int, int8, int16, int32, int64: + return float64(src.Elem().Int()), nil + case uint, uint8, uint16, uint32, uint64: + return float64(src.Elem().Uint()), nil + case float32, float64: + return float64(src.Elem().Float()), nil + case complex64, complex128: + return real(src.Elem().Complex()), nil + case string: + s, _ := strconv.ParseFloat(src.Elem().String(), 64) + return s, nil + case *float64: + s := *src.Interface().(*float64) + return s, nil + } + + return 0, fmt.Errorf("src cannot be converted") +} + +func asComplex(src reflect.Value) (complex128, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return 0, fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case int, int8, int16, int32, int64: + return complex(float64(src.Elem().Int()), 0), nil + case uint, uint8, uint16, uint32, uint64: + return complex(float64(src.Elem().Uint()), 0), nil + case float32, float64: + return complex(float64(src.Elem().Float()), 0), nil + case complex64, complex128: + return src.Elem().Complex(), nil + case string: + s, _ := strconv.ParseFloat(src.Elem().String(), 64) + return complex(s, 0), nil + case *complex128: + s := *src.Interface().(*complex128) + return s, nil + } + + return 0, fmt.Errorf("src cannot be converted") +} + +func asString(src reflect.Value) (string, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return "", fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case int, int8, int16, int32, int64: + return fmt.Sprintf("%v", src.Elem().Int()), nil + case uint, uint8, uint16, uint32, uint64: + return fmt.Sprintf("%v", src.Elem().Uint()), nil + case float32, float64: + return fmt.Sprintf("%v", src.Elem().Float()), nil + case complex64, complex128: + return fmt.Sprintf("%v", src.Elem().Complex()), nil + case string: + return src.Elem().String(), nil + case []byte: + return string(src.Elem().Bytes()), nil + case *string: + s := *src.Interface().(*string) + return s, nil + } + + return "", fmt.Errorf("src cannot be converted") +} + +func asTime(src reflect.Value) (time.Time, error) { + if src.Kind() != reflect.Interface && src.Kind() != reflect.Ptr { + return time.Time{}, fmt.Errorf("src must be Interface or Ptr") + } + + switch src.Interface().(type) { + case int, int8, int16, int32, int64: + return time.Unix(src.Elem().Int(), 0), nil + case uint, uint8, uint16, uint32, uint64: + return time.Unix(int64(src.Elem().Uint()), 0), nil + case string: + if s, err := time.Parse(time.RFC3339, src.Elem().String()); err == nil { + return s, nil + } + if s, err := time.Parse("2006-01-02 15:04:05", src.Elem().String()); err == nil { + return s, nil + } + if s, err := time.Parse("2006-01-02", src.Elem().String()); err == nil { + return s, nil + } + if s, err := time.Parse("15:04:05", src.Elem().String()); err == nil { + return s, nil + } + case time.Time: + return src.Elem().Interface().(time.Time), nil + case *time.Time: + s := *src.Interface().(*time.Time) + return s, nil + } + + return time.Time{}, fmt.Errorf("src cannot be converted") +} diff --git a/utils/convutil/sort.go b/utils/convutil/sort.go new file mode 100644 index 0000000..7110d26 --- /dev/null +++ b/utils/convutil/sort.go @@ -0,0 +1,95 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package convutil + +import ( + "reflect" + "sort" +) + +// SortMap returns an sorted array of map keys. +func SortMap(v interface{}) []string { + vv := reflect.ValueOf(v) + if vv.Kind() != reflect.Map { + return nil + } + + keys := make([]string, 0) + for _, k := range vv.MapKeys() { + keys = append(keys, k.String()) + } + return keys +} + +// SortMapInt returns an sorted int array of map keys. +func SortMapInt(v interface{}) []int { + vv := reflect.ValueOf(v) + if vv.Kind() != reflect.Map { + return nil + } + vks := vv.MapKeys() + if len(vks) <= 0 { + return nil + } + + keys := make([]int, 0) + switch vks[0].Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + for _, k := range vks { + keys = append(keys, int(k.Int())) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + for _, k := range vks { + keys = append(keys, int(k.Uint())) + } + } + + sort.Ints(keys) + return keys +} + +// SortMapFloat64 returns an sorted float array of map keys. +func SortMapFloat64(v interface{}) []float64 { + vv := reflect.ValueOf(v) + if vv.Kind() != reflect.Map { + return nil + } + vks := vv.MapKeys() + if len(vks) <= 0 { + return nil + } + + keys := make([]float64, 0) + switch vks[0].Kind() { + case reflect.Float32, reflect.Float64: + for _, k := range vks { + keys = append(keys, k.Float()) + } + } + + sort.Float64s(keys) + return keys +} + +// SortMapString returns an sorted string array of map keys. +func SortMapString(v interface{}) []string { + vv := reflect.ValueOf(v) + if vv.Kind() != reflect.Map { + return nil + } + vks := vv.MapKeys() + if len(vks) <= 0 { + return nil + } + + keys := make([]string, 0) + switch vks[0].Kind() { + case reflect.String: + for _, k := range vks { + keys = append(keys, k.String()) + } + } + + sort.Strings(keys) + return keys +} diff --git a/utils/dbutil/format.go b/utils/dbutil/format.go new file mode 100644 index 0000000..0285ec2 --- /dev/null +++ b/utils/dbutil/format.go @@ -0,0 +1,499 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package dbutil + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/fatih/structs" +) + +// FormatCtx define parameters for formatting query/stats/serie string. +type FormatCtx struct { + TableName string + UseCount bool // for query + Wheres string // for stats +} + +// Query defines request query object. +type Query struct { + Fields []string + Wheres map[string]string + Groupby string + OrderBy string + Limits map[string]int64 + Params map[string]string +} + +// Parse returns request list to Query. +func (query *Query) Parse(params interface{}) { + switch params.(type) { + case map[string]interface{}: + default: + return + } + for key, value := range params.(map[string]interface{}) { + switch strings.ToLower(key) { + case "fields": + query.Fields = make([]string, 0) + switch value.(type) { + case string: + query.Fields = append(query.Fields, value.(string)) + case []interface{}: + for _, v := range value.([]interface{}) { + switch v.(type) { + case string: + query.Fields = append(query.Fields, v.(string)) + } + } + } + case "wheres": + query.Wheres = make(map[string]string) + switch value.(type) { + case map[string]interface{}: + for k, v := range value.(map[string]interface{}) { + switch v.(type) { + case string: + query.Wheres[k] = fmt.Sprintf("%s", v.(string)) + case []interface{}: + vt := make([]string, 0) + for _, t := range v.([]interface{}) { + switch t.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", t.(string))) + } + } + if len(vt) >= 2 { + query.Wheres[k] = fmt.Sprintf("BETWEEN %s AND %s", vt[0], vt[1]) + } + case map[string]interface{}: + for _, t := range v.(map[string]interface{}) { + switch t.(type) { + case []interface{}: + vt := make([]string, 0) + for _, p := range t.([]interface{}) { + switch p.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", p.(string))) + } + } + if len(vt) > 0 { + query.Wheres[k] = fmt.Sprintf("IN(%s)", strings.Join(vt, ",")) + } + } + break + } + } + } + } + case "groupby": + switch value.(type) { + case string: + query.Groupby = value.(string) + } + case "orderby": + switch value.(type) { + case string: + query.OrderBy = value.(string) + } + case "limits": + query.Limits = make(map[string]int64) + switch value.(type) { + case map[string]interface{}: + for k, v := range value.(map[string]interface{}) { + switch strings.ToLower(k) { + case "offset": + query.Limits["offset"], _ = strconv.ParseInt(fmt.Sprint(v), 0, 64) + case "limit": + query.Limits["limit"], _ = strconv.ParseInt(fmt.Sprint(v), 0, 64) + } + } + } + case "params": + query.Params = make(map[string]string) + switch value.(type) { + case map[string]interface{}: + for k, v := range value.(map[string]interface{}) { + query.Params[k] = fmt.Sprint(v) + } + } + } + } +} + +// Format return formatted query string using table name +func (query *Query) Format(ctx FormatCtx) string { + fields := "*" + if len(query.Fields) > 0 { + fields = strings.Join(query.Fields, ",") + } + wheres := "" + conditions := make([]string, 0) + if query.Wheres != nil { + for key := range query.Wheres { + conditions = append(conditions, fmt.Sprintf("%s %s", key, query.Wheres[key])) + } + } + if ctx.Wheres != "" { + conditions = append(conditions, ctx.Wheres) + } + if len(conditions) > 0 { + wheres = fmt.Sprintf(" WHERE %s", strings.Join(conditions, " AND ")) + } + groupby := "" + if query.Groupby != "" { + groupby = fmt.Sprintf(" GROUP BY %s ", query.Groupby) + } + orderby := "" + if query.OrderBy != "" { + orderby = fmt.Sprintf(" ORDER BY %s", query.OrderBy) + } + limits := "" + if query.Limits["offset"] > 0 { + limits += fmt.Sprintf(" OFFSET %d", query.Limits["offset"]) + } + if query.Limits["limit"] > 0 { + limits += fmt.Sprintf(" LIMIT %d", query.Limits["limit"]) + } + + var sql string + if ctx.UseCount { + sql = "SELECT COUNT(1) FROM " + ctx.TableName + wheres + } else { + sql = "SELECT " + fields + " FROM " + ctx.TableName + wheres + groupby + orderby + limits + } + return sql +} + +// Stats defines request stats object. +type Stats struct { + Name string + Function string + Timezone string + Series []string + Wheres map[string]string +} + +// StatsRes defines response stats object. +type StatsRes struct { + Minute *interface{} `db:"minute"` + Hour *interface{} `db:"hour"` + Day *interface{} `db:"day"` + Week *interface{} `db:"week"` + Month *interface{} `db:"month"` + Quarter *interface{} `db:"quarter"` + Year *interface{} `db:"year"` + All *interface{} `db:"all"` +} + +// Parse returns request list to Stats. +func (stats *Stats) Parse(params interface{}) { + switch params.(type) { + case map[string]interface{}: + default: + return + } + + for key, value := range params.(map[string]interface{}) { + switch strings.ToLower(key) { + case "name": + stats.Name = fmt.Sprint(value) + case "function": + stats.Function = strings.ToLower(fmt.Sprint(value)) + case "timezone": + stats.Timezone = fmt.Sprint(value) + case "series": + switch value.(type) { + case string: + stats.Series = append(stats.Series, value.(string)) + case []interface{}: + for _, v := range value.([]interface{}) { + switch v.(type) { + case string: + stats.Series = append(stats.Series, v.(string)) + } + } + } + case "wheres": + stats.Wheres = make(map[string]string) + switch value.(type) { + case map[string]interface{}: + for k, v := range value.(map[string]interface{}) { + switch v.(type) { + case string: + stats.Wheres[k] = fmt.Sprintf("%s", v.(string)) + case []interface{}: + vt := make([]string, 0) + for _, t := range v.([]interface{}) { + switch t.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", t.(string))) + } + } + if len(vt) >= 2 { + stats.Wheres[k] = fmt.Sprintf("BETWEEN %s AND %s", vt[0], vt[1]) + } + case map[string]interface{}: + for _, t := range v.(map[string]interface{}) { + switch t.(type) { + case []interface{}: + vt := make([]string, 0) + for _, p := range t.([]interface{}) { + switch p.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", p.(string))) + } + } + if len(vt) > 0 { + stats.Wheres[k] = fmt.Sprintf("IN(%s)", strings.Join(vt, ",")) + } + } + break + } + } + } + } + } + } + + if stats.Function == "count" { + stats.Name = "1" + } + if stats.Timezone == "" { + stats.Timezone = "+8" // UTC+8 + } +} + +// Format return formatted stats string using table name +func (stats *Stats) Format(ctx FormatCtx) string { + if len(stats.Series) <= 0 { + return "" + } + wheres := "" + conditions := make([]string, 0) + if stats.Wheres != nil { + for key := range stats.Wheres { + conditions = append(conditions, fmt.Sprintf("%s %s", key, stats.Wheres[key])) + } + } + if ctx.Wheres != "" { + conditions = append(conditions, ctx.Wheres) + } + if len(conditions) > 0 { + wheres = fmt.Sprintf(" WHERE %s", strings.Join(conditions, " AND ")) + } + + var sql string + var template string + template = fmt.Sprintf("SELECT %v(%v) FROM %v %v", stats.Function, stats.Name, ctx.TableName, wheres) + totals := make([]string, 0) + for _, v := range stats.Series { + key := strings.ToLower(v) + switch key { + case "minute", "hour", "day", "week", "month", "quarter", "year": + totals = append(totals, fmt.Sprintf("(%v AND (created AT TIME ZONE '%v')>=date_trunc('%s', LOCALTIMESTAMP AT TIME ZONE '%v')) AS %s", template, stats.Timezone, key, stats.Timezone, v)) + case "all": + totals = append(totals, fmt.Sprintf("(%v) AS %s", template, v)) + } + } + sql = "SELECT " + for i, v := range totals { + if i > 0 { + sql += "," + } + sql += v + } + return sql +} + +// Serie defines request serie object. +type Serie struct { + Name string + Function string + Timezone string + Start string + Stop string + Step string + Wheres map[string]string +} + +// SerieRes defines response serie object. +type SerieRes struct { + Field []interface{} + Value []interface{} +} + +// Parse returns request list to Stats. +func (serie *Serie) Parse(params interface{}) { + switch params.(type) { + case map[string]interface{}: + default: + return + } + + for key, value := range params.(map[string]interface{}) { + switch strings.ToLower(key) { + case "name": + serie.Name = fmt.Sprint(value) + case "function": + serie.Function = strings.ToLower(fmt.Sprint(value)) + case "timezone": + serie.Timezone = fmt.Sprint(value) + case "start": + serie.Start = fmt.Sprint(value) + case "stop": + serie.Stop = fmt.Sprint(value) + case "step": + serie.Step = fmt.Sprint(value) + case "wheres": + serie.Wheres = make(map[string]string) + switch value.(type) { + case map[string]interface{}: + for k, v := range value.(map[string]interface{}) { + switch v.(type) { + case string: + serie.Wheres[k] = fmt.Sprintf("%s", v.(string)) + case []interface{}: + vt := make([]string, 0) + for _, t := range v.([]interface{}) { + switch t.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", t.(string))) + } + } + if len(vt) >= 2 { + serie.Wheres[k] = fmt.Sprintf("BETWEEN %s AND %s", vt[0], vt[1]) + } + case map[string]interface{}: + for _, t := range v.(map[string]interface{}) { + switch t.(type) { + case []interface{}: + vt := make([]string, 0) + for _, p := range t.([]interface{}) { + switch p.(type) { + case string: + vt = append(vt, fmt.Sprintf("'%s'", p.(string))) + } + } + if len(vt) > 0 { + serie.Wheres[k] = fmt.Sprintf("IN(%s)", strings.Join(vt, ",")) + } + } + break + } + } + } + } + } + } + + if serie.Function == "count" { + serie.Name = "1" + } + if serie.Timezone == "" { + serie.Timezone = "+8" // UTC+8 + } +} + +// Format return formatted stats string using table name +func (serie *Serie) Format(ctx FormatCtx) string { + if len(serie.Step) <= 0 { + return "" + } + wheres := "" + conditions := make([]string, 0) + if serie.Wheres != nil { + for key := range serie.Wheres { + conditions = append(conditions, fmt.Sprintf("%s %s", key, serie.Wheres[key])) + } + } + if ctx.Wheres != "" { + conditions = append(conditions, ctx.Wheres) + } + if len(conditions) > 0 { + wheres = fmt.Sprintf(" WHERE %s", strings.Join(conditions, " AND ")) + } + + re := regexp.MustCompile(`(^[0-9 ]*|[ s]*)`) + key := re.ReplaceAllString(strings.ToLower(serie.Step), "") + num := 10 + format := "YYYY" + switch key { + case "minute": + num = 59 + format = "YYYY-MM-DD HH:MI" + case "hour": + num = 23 + format = "YYYY-MM-DD HH:00" + case "day": + num = 29 + format = "YYYY-MM-DD" + case "week": + num = 53 + format = "YYYY-WW" + case "month": + num = 11 + format = "YYYY-MM" + case "quarter": + num = 33 + format = "YYYY-Q" + key = "month" + serie.Step = "3 month" + case "year": + num = 9 + format = "YYYY" + } + format = fmt.Sprintf("'%v'", format) + if serie.Start == "" { + serie.Start = fmt.Sprintf("LOCALTIMESTAMP-interval '%d %s'", num, key) + } else { + serie.Start = fmt.Sprintf("TIMESTAMP '%v'", serie.Start) + } + serie.Start = fmt.Sprintf("(%v) AT TIME ZONE '%v'", serie.Start, serie.Timezone) + if serie.Stop == "" { + serie.Stop = "LOCALTIMESTAMP" + } else { + serie.Stop = fmt.Sprintf("TIMESTAMP '%v'", serie.Stop) + } + serie.Stop = fmt.Sprintf("(%v) AT TIME ZONE '%v'", serie.Stop, serie.Timezone) + if serie.Step[0] != '\'' { + serie.Step = fmt.Sprintf("'%v'", serie.Step) + } + + sql := fmt.Sprintf(` +SELECT x.v AS key, COALESCE(y.v, 0) AS value FROM ( + SELECT to_char(s.t, %v) AS v FROM ( + SELECT generate_series(%v, %v, %v) + ) AS s(t)) + AS x +LEFT JOIN ( + SELECT to_char(created AT TIME ZONE '%v', %v) AS k, %v(%v) AS v FROM %v %v GROUP BY k +) AS y ON x.v = y.k; +`, format, serie.Start, serie.Stop, serie.Step, serie.Timezone, format, serie.Function, serie.Name, ctx.TableName, wheres) + return sql +} + +// Wheres returns an formatted sql where string without 'WHERE' from obj's value, +// exp: {ad_client_id: '1', name: 'test'} => "ad_client_id='1' AND name='test'" +func Wheres(obj interface{}) string { + var wheres string + m := make(map[string]interface{}) + switch obj.(type) { + case map[string]interface{}: + m = obj.(map[string]interface{}) + default: + m = structs.Map(obj) + } + conditions := make([]string, 0) + for key, value := range m { + conditions = append(conditions, fmt.Sprintf("%v=%v", key, value)) + } + if len(conditions) > 0 { + wheres = strings.Join(conditions, " AND ") + } + + return wheres +} diff --git a/utils/dsutil/array.go b/utils/dsutil/array.go new file mode 100644 index 0000000..ca9710d --- /dev/null +++ b/utils/dsutil/array.go @@ -0,0 +1,168 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package dsutil + +import ( + "reflect" +) + +// Equal returns true if array equals to val. +func Equal(array interface{}, val interface{}) bool { + va, vv := reflect.ValueOf(array), reflect.ValueOf(val) + if (va.Kind() != reflect.Array && va.Kind() != reflect.Slice) || + (vv.Kind() != reflect.Array && vv.Kind() != reflect.Slice) { + return false + } + + na, nv := va.Len(), vv.Len() + if na != nv { + return false + } + + i := 0 + for i = 0; i < na; i++ { + if va.Index(i).Interface() != vv.Index(i).Interface() { + break + } + } + return i == na +} + +// IncludesAt returns index if array includes val. +func IncludesAt(array interface{}, val interface{}) int { + va, vv := reflect.ValueOf(array), reflect.ValueOf(val) + if (va.Kind() != reflect.Array && va.Kind() != reflect.Slice) || 0 >= va.Len() { + return -1 + } + + if vv.Kind() == reflect.Array || vv.Kind() == reflect.Slice { + if 0 >= vv.Len() || reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(vv.Index(0).Interface()) { + return -1 + } + + alen, vlen := va.Len(), vv.Len() + var i, j int + for i = 0; i <= alen-vlen; i++ { + for j = 0; j < vlen; j++ { + if va.Index(i+j).Interface() != vv.Index(j).Interface() { + break + } + } + if j == vlen { + return i + } + } + } else { + if reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(val) { + return -1 + } + + for i := 0; i < va.Len(); i++ { + if va.Index(i).Interface() == val { + return i + } + } + } + + return -1 +} + +// ContainsAny returns true if array contains any elements in val. +func ContainsAny(array interface{}, val interface{}) bool { + i, _ := ContainsAnyAt(array, val) + return i != -1 +} + +// ContainsAnyAt returns index if array contains any elements in val. +func ContainsAnyAt(array interface{}, val interface{}) (int, int) { + va, vv := reflect.ValueOf(array), reflect.ValueOf(val) + if (va.Kind() != reflect.Array && va.Kind() != reflect.Slice) || 0 >= va.Len() { + return -1, -1 + } + + if vv.Kind() == reflect.Array || vv.Kind() == reflect.Slice { + if 0 >= vv.Len() || reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(vv.Index(0).Interface()) { + return -1, -1 + } + + for i := 0; i < va.Len(); i++ { + for j := 0; j < vv.Len(); j++ { + if va.Index(i).Interface() == vv.Index(j).Interface() { + return i, j + } + } + } + } else { + if reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(val) { + return -1, -1 + } + + for i := 0; i < va.Len(); i++ { + if va.Index(i).Interface() == val { + return i, 0 + } + } + } + + return -1, -1 +} + +// ContainsAll returns true if array contains all elements in val. +func ContainsAll(array interface{}, val interface{}) bool { + res := ContainsAllAt(array, val) + return res != nil +} + +// ContainsAllAt returns index if array contains all elements in val. +func ContainsAllAt(array interface{}, val interface{}) []int { + va, vv := reflect.ValueOf(array), reflect.ValueOf(val) + if (va.Kind() != reflect.Array && va.Kind() != reflect.Slice) || 0 >= va.Len() { + return nil + } + + if vv.Kind() == reflect.Array || vv.Kind() == reflect.Slice { + if 0 >= vv.Len() || reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(vv.Index(0).Interface()) { + return nil + } + + alen, vlen := va.Len(), vv.Len() + used, res := []bool{}, []int{} + for i := 0; i < alen; i++ { + used = append(used, false) + } + for i := 0; i < vlen; i++ { + res = append(res, -1) + } + + var i, j int + for i = 0; i < vlen; i++ { + for j = 0; j < alen; j++ { + if used[j] { + continue + } + if va.Index(j).Interface() == vv.Index(i).Interface() { + used[j] = true + res[i] = j + break + } + } + if j == alen { + return nil + } + } + if i == vlen { + return res + } + } else { + if reflect.TypeOf(va.Index(0).Interface()) != reflect.TypeOf(val) { + return nil + } + + for i := 0; i < va.Len(); i++ { + if va.Index(i).Interface() == val { + return []int{i} + } + } + } + return nil +} diff --git a/utils/fmtutil/format.go b/utils/fmtutil/format.go new file mode 100644 index 0000000..efb2170 --- /dev/null +++ b/utils/fmtutil/format.go @@ -0,0 +1,55 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package fmtutil + +import ( + "fmt" + "math" + "strconv" + "time" +) + +// FormatDate returns a string formatted value with type: date.storage. +func FormatDate(value interface{}, timezone, format string) string { + var s time.Time + switch value.(type) { + case string: + s, _ = time.Parse(time.RFC3339, value.(string)) + case *time.Time: + s = *value.(*time.Time) + } + + var options string + switch format { + case "date": + options = "2006/01/02" + case "time": + options = "15:04:05" + default: + options = "2006/01/02 15:04:05" + } + + if timezone == "" { + timezone = "Asia/Shanghai" + } + tz, _ := time.LoadLocation(timezone) + return s.In(tz).Format(options) +} + +// FormatStorage returns a string formatted value with type: uom.storage. +func FormatStorage(value interface{}) string { + var bytes int64 + switch value.(type) { + case int64: + bytes = value.(int64) + case *string: + bytes, _ = strconv.ParseInt(*value.(*string), 0, 64) + } + + if bytes == 0 { + return "0 B" + } + units := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} + i := math.Floor(math.Log(float64(bytes)) / math.Log(1024)) + return fmt.Sprintf("%.2f %v", (float64(bytes) / math.Pow(1024, i)), units[int(i)]) +} diff --git a/utils/fsutil/archive.go b/utils/fsutil/archive.go new file mode 100644 index 0000000..1f5f692 --- /dev/null +++ b/utils/fsutil/archive.go @@ -0,0 +1,184 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package fsutil + +import ( + "archive/zip" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "git.uzoombox.com/git/uzsdk/utils/osutil" +) + +// ZipFilepath use zip to archive a path(s) and output to zipfile, +// if successful, it return nil, else return an error. +// options will be [recursive, files]: +// +// recursive bool: zip directory recursively, default: true. +// files []string: only zip these files, default zip all files. +func ZipFilepath(paths []string, target string, options ...interface{}) error { + var recursive = true + var files []string + for _, option := range options { + switch option.(type) { + case bool: + recursive = option.(bool) + case []string: + files = option.([]string) + } + } + + zf, err := os.Create(target) + if err != nil { + return err + } + defer zf.Close() + + zw := zip.NewWriter(zf) + defer zw.Close() + + if len(files) > 0 { // only zip those file that found in 'files'. + for _, file := range files { + var path string + for i := 0; i < len(paths); i++ { + if strings.HasPrefix(file, paths[i]) && len(paths[i]) > len(path) { + path = paths[i] + } + } + if osutil.IsDir(file) { + filepath.WalkDir(file, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !recursive { // do not zip recursively. + if d.IsDir() && file != p { + return filepath.SkipDir + } + } + return zipFilenode(zw, path, p) + }) + } else { + zipFilenode(zw, path, file) + } + } + } else { // zip all files. + for _, path := range paths { + filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !recursive { // do not zip recursively. + if d.IsDir() && path != p { + return filepath.SkipDir + } + } + return zipFilenode(zw, path, p) + }) + } + } + + return nil +} + +// UnzipFilepath use zip to unarchive a path and output to target directory, +// if successful, it return nil, else return an error. +func UnzipFilepath(path, target string) error { + zr, err := zip.OpenReader(path) + if err != nil { + return err + } + defer zr.Close() + + root, err := filepath.Abs(target) + if err != nil { + return err + } + + for _, zf := range zr.File { + if err = unzipFilenode(zr, root, zf); err != nil { + return err + } + } + + return nil +} + +func zipFilenode(w *zip.Writer, root, path string) error { + // create a local file header. + fi, err := os.Stat(path) + if err != nil { + return err + } + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + fh.Method = zip.Deflate + fh.Name, err = filepath.Rel(filepath.Dir(root), path) + if err != nil { + return err + } + if fi.IsDir() { + fh.Name += "/" + } + + // create writer for this file header. + fw, err := w.CreateHeader(fh) + if err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + // save content of the file. + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(fw, f) + return err +} + +func unzipFilenode(_ *zip.ReadCloser, root string, zf *zip.File) error { + path := filepath.Join(root, zf.Name) + if !strings.HasPrefix(path, filepath.Clean(root)+string(os.PathSeparator)) { + return fmt.Errorf("invalid file path: %s", path) + } + + if zf.FileInfo().IsDir() { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + return nil + } + + // create directory tree. + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + // create a local file for receiving unzipped content. + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + // unzip the content of zipfile and copy to local file. + fr, err := zf.Open() + if err != nil { + return err + } + defer fr.Close() + + _, err = io.Copy(f, fr) + return err +} diff --git a/utils/fsutil/crypto.go b/utils/fsutil/crypto.go new file mode 100644 index 0000000..d6b3949 --- /dev/null +++ b/utils/fsutil/crypto.go @@ -0,0 +1,166 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package fsutil + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "fmt" + "io" + "os" + "path/filepath" + "sort" +) + +// Md5sum returns the MD5 checksum of file path. +func Md5sum(path string) (sum string) { + fi, err := os.Stat(path) + if err != nil { + return + } + + if fi.IsDir() { + if false { + names := make([]string, 0) + filepath.WalkDir(path, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + names = append(names, p) + if d.IsDir() && path != p { + return filepath.SkipDir + } + return nil + }) + + sort.Strings(names) + sum = Md5string(names...) + } else { + sum = "" + } + } else { + sum = Md5file(path) + } + + return +} + +// Md5file returns the MD5 checksum of a file. +func Md5file(path string) string { + var sum string + if f, err := os.Open(path); err == nil { + h := md5.New() + if _, err := io.Copy(h, f); err == nil { + sum = fmt.Sprintf("%x", h.Sum(nil)) + } + f.Close() + } + return sum +} + +// Md5string returns the MD5 checksum of string(s). +func Md5string(ss ...string) string { + h := md5.New() + for _, s := range ss { + io.WriteString(h, s) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// AesEncrypt use keyval to encrypt data with AES, and returns the result. +func AesEncrypt(data, keyval []byte) []byte { + var dst []byte + key := Aes256Key(keyval) + block, err := aes.NewCipher(key) + if err != nil { + return dst + } + + defer func() { + if err := recover(); err != nil { + fmt.Println("panic occurred:", err) + } + }() + + blocksize := block.BlockSize() + padding := blocksize - len(data)%blocksize + src := append(data, bytes.Repeat([]byte{byte(padding)}, padding)...) // pkcs#7 + dst = make([]byte, len(src)) + mode := cipher.NewCBCEncrypter(block, key[:blocksize]) // iv=key[:blocksize] + mode.CryptBlocks(dst, src) + return dst +} + +// AesDecrypt use keyval to decrypt data with AES, and returns the result. +func AesDecrypt(data, keyval []byte) []byte { + var dst []byte + key := Aes256Key(keyval) + block, err := aes.NewCipher(key) + if err != nil { + return dst + } + + defer func() { + if err := recover(); err != nil { + fmt.Println("panic occurred:", err) + } + }() + + blocksize := block.BlockSize() + dstlen := len(data) + dst = make([]byte, dstlen) + mode := cipher.NewCBCDecrypter(block, key[:blocksize]) + mode.CryptBlocks(dst, data) + unpadding := int(dst[dstlen-1]) + return dst[:(dstlen - unpadding)] // pkcs#7 +} + +// AesCryptfile use keyval to encrypt/decrypt file with AES, and returns the result. +func AesCryptfile(src, dst string, keyval []byte) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return err + } + defer out.Close() + + key := Aes256Key(keyval) + block, err := aes.NewCipher(key) + if err != nil { + return err + } + + defer func() { + if err := recover(); err != nil { + fmt.Println("panic occurred:", err) + } + }() + + blocksize := block.BlockSize() + stream := cipher.NewOFB(block, key[:blocksize]) // iv=key[:blocksize] + reader := &cipher.StreamReader{S: stream, R: in} + _, err = io.Copy(out, reader) + return err +} + +// Aes256Key returns a key with 256-bits used for aes from val. +func Aes256Key(val []byte) []byte { + keylen := 32 + key := make([]byte, 0) + namelen := len(val) + for len(key) < keylen { + left := keylen - len(key) + addlen := left + if left > namelen { + addlen = namelen + } + key = append(key, []byte(val[:addlen])...) + } + return key +} diff --git a/utils/fsutil/encoding.go b/utils/fsutil/encoding.go new file mode 100644 index 0000000..2b2d9c8 --- /dev/null +++ b/utils/fsutil/encoding.go @@ -0,0 +1,64 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package fsutil + +import ( + "encoding/base64" + "log" + "os" +) + +// Base64file returns the base64 encoded string of a file. +func Base64file(path string) string { + var result string + if bytes, err := os.ReadFile(path); err == nil { + result = base64.StdEncoding.EncodeToString(bytes) + } + return result +} + +// Base64Encode returns the base64 encoded string of a text, +// options will be [noPadding]: +// +// noPadding bool: not padding with '=' if set to true, default: false. +func Base64Encode(text string, options ...interface{}) string { + var noPadding bool + for _, option := range options { + switch option.(type) { + case bool: + noPadding = option.(bool) + } + } + + if noPadding { + return base64.RawStdEncoding.EncodeToString([]byte(text)) + } else { + return base64.StdEncoding.EncodeToString([]byte(text)) + } +} + +// Base64Decode returns the decoded string with base64 of a text, +// options will be [noPadding]: +// +// noPadding bool: not padding with '=' if set to true, default: false. +func Base64Decode(text string, options ...interface{}) string { + var noPadding bool + for _, option := range options { + switch option.(type) { + case bool: + noPadding = option.(bool) + } + } + + buf, err := []byte{}, error(nil) + if noPadding { + buf, err = base64.RawStdEncoding.DecodeString(text) + } else { + buf, err = base64.StdEncoding.DecodeString(text) + } + if err != nil { + log.Println(err) + } + + return string(buf) +} diff --git a/utils/fsutil/file.go b/utils/fsutil/file.go new file mode 100644 index 0000000..8af44c9 --- /dev/null +++ b/utils/fsutil/file.go @@ -0,0 +1,128 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package fsutil + +import ( + "os" + "regexp" + "strings" +) + +// Filetype returns filetype by mimetype like:[ +// +// document, presentation, spreadsheet, package, pdf, +// audio, image, text, video, chemical, font, message, model, application, +// folder, httpd, other] +// +// read more from https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +func Filetype(name string) string { + types := strings.Split(name, "/") + if len(types) > 1 { + switch types[0] { + case "application": + { + if matched, _ := regexp.MatchString(`(ms[-]?word)|(wordprocessingml)|(opendocument\.text)|(abiword)`, types[1]); matched { + return "document" + } else if matched, _ := regexp.MatchString(`(ms[-]?powerpoint)|(presentationml)|(opendocument\.presentation)`, types[1]); matched { + return "presentation" + } else if matched, _ := regexp.MatchString(`(ms[-]?excel)|(spreadsheetml)|(opendocument\.spreadsheet)`, types[1]); matched { + return "spreadsheet" + } else if matched, _ := regexp.MatchString(`((archive|compressed|cd-image|diskimage|freearc|rar|tar|zip2?)$)|(\.installer)`, types[1]); matched { + return "package" + } else if matched, _ := regexp.MatchString(`^pdf$`, types[1]); matched { + return "pdf" + } else if matched, _ := regexp.MatchString(`(^(\w+script|(ld\+)?json|(xhtml\+)?xml|rtf|sql|x-c?sh|x-php|x-subrip|x-yaml)$)|((\.xul\+)+xml$)`, types[1]); matched { + return "text" + } else if matched, _ := regexp.MatchString(`fontobject$`, types[1]); matched { + return "font" + } else if matched, _ := regexp.MatchString(`^ogg$`, types[1]); matched { + return "video" + } else { + return "application" + } + } + case "audio": + return "audio" + case "chemical": + return "chemical" + case "font": + return "font" + case "httpd": + { + if types[1] == "unix-directory" { + return "folder" + } else { + return "httpd" + } + } + case "image": + return "image" + case "message": + return "message" + case "model": + return "model" + case "text": + return "text" + case "video": + return "video" + } + } + + return "other" +} + +// Filemime returns regexp string of mimetype by filetype, exp: +// +// document => (ms[-]?word)|(wordprocessingml)|(opendocument\.text)|(abiword) +// +// read more from https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +func Filemime(name string) string { + switch strings.ToLower(name) { + case "document": + return `^application/.+((ms[-]?word)|(wordprocessingml)|(opendocument\.text)|(abiword))` + case "presentation": + return `^application/.+((ms[-]?powerpoint)|(presentationml)|(opendocument\.presentation))` + case "spreadsheet": + return `^application/.+((ms[-]?excel)|(spreadsheetml)|(opendocument\.spreadsheet))` + case "package": + return `^application/.*(((archive|compressed|cd-image|diskimage|freearc|rar|tar|zip2?)$)|(\.installer))` + case "pdf": + return `^application/pdf$` + case "application": + return `^application/` + case "audio": + return `^audio/` + case "chemical": + return `^chemical/` + case "font": + return `^application/.*(fontobject$)` + `|` + `^font/` + case "folder": + return `^httpd/unix-directory` + case "httpd": + return `^httpd/(?!unix-directory).` + case "image": + return `^image/` + case "message": + return `^message/` + case "model": + return `^model/` + case "text": + return `^application/.*(((\w+script|(ld\+)?json|(xhtml\+)?xml|rtf|sql|x-c?sh|x-php|x-subrip|x-yaml)$)|((\.xul\+)+xml$))` + `|` + `^text/` + case "video": + return `^application/(ogg$)` + `|` + `^video/` + case "other": + return `^(?!application|audio|chemical|font|httpd|image|message|model|text|video).` + } + + return `^.{0}$` +} + +// Filesize returns filesize of the regular file(following symbolic link). +func Filesize(path string) int64 { + fi, err := os.Stat(path) + if err != nil { + return -1 + } + + return fi.Size() +} diff --git a/utils/netutil/mime.go b/utils/netutil/mime.go new file mode 100644 index 0000000..d043494 --- /dev/null +++ b/utils/netutil/mime.go @@ -0,0 +1,65 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package netutil + +import ( + "io/fs" + "mime" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" +) + +// Mimetype returns the MIME type associated with the file path, +// if path is a directory, returns "httpd/unix-directory". +// Parameters WOULD include [fi:string/fs.FileInfo, re:*regexp.Regexp] +func Mimetype(options ...interface{}) string { + var fi fs.FileInfo + var re *regexp.Regexp + for _, option := range options { + switch option.(type) { + case string: + fi, _ = os.Stat(option.(string)) + case fs.FileInfo: + fi = option.(fs.FileInfo) + case *regexp.Regexp: + re = option.(*regexp.Regexp) + } + } + + if fi.IsDir() { + return "httpd/unix-directory" + } + path := fi.Name() + if re != nil { + if loc := re.FindStringIndex(path); loc != nil { + path = path[:loc[0]] + } + } + return mime.TypeByExtension(filepath.Ext(path)) +} + +// Multipart returns a multipart string that split from Header.Content-Type, +// options will be [content]: +// +// content [*http.Request, *string]: header["Content-Type"] +func Multipart(options ...interface{}) string { + var content string // "Content-Type: multipart/form-data;boundary=---yb1zYhTI38xpQxBK" + for _, option := range options { + switch option.(type) { + case *http.Request: + content = option.(*http.Request).Header.Get("Content-Type") + case *string: + content = *option.(*string) + } + } + + for _, part := range strings.Split(content, ";") { + if strings.Contains(part, "multipart") { + return part + } + } + return "" +} diff --git a/utils/osutil/exec.go b/utils/osutil/exec.go new file mode 100644 index 0000000..e84acc6 --- /dev/null +++ b/utils/osutil/exec.go @@ -0,0 +1,43 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package osutil + +import ( + "io" + "os/exec" +) + +// ExecCommand construct a command struct and then call cmd.Start and cmd.Wait, +// returns the command struct and exec status at last. +func ExecCommand(name string, arg ...string) (*exec.Cmd, error) { + cmd := exec.Command(name, arg...) + err := cmd.Start() + if err != nil { + return cmd, err + } + + return cmd, cmd.Wait() +} + +// ReadCommandOutput returns command's exec status, stdout and stderr buffer. +func ReadCommandOutput(name string, arg ...string) (err error, obuf, ebuf []byte) { + cmd := exec.Command(name, arg...) + o, err := cmd.StdoutPipe() + if err != nil { + return err, nil, nil + } + e, err := cmd.StderrPipe() + if err != nil { + return err, nil, nil + } + + err = cmd.Start() + if err != nil { + return err, nil, nil + } + obuf, err = io.ReadAll(o) + ebuf, err = io.ReadAll(e) + + cmd.Wait() + return err, obuf, ebuf +} diff --git a/utils/osutil/path.go b/utils/osutil/path.go new file mode 100644 index 0000000..6a8913b --- /dev/null +++ b/utils/osutil/path.go @@ -0,0 +1,107 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package osutil + +import ( + "io/fs" + "log" + "os" + "path/filepath" + "regexp" +) + +// GetAppDirectory returns directory where the application located, NOT working directory!!! +// Use os.Getwd to get current working directory. +func GetAppDirectory() string { + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + log.Fatal(err) + } + return dir +} + +// CreateDirectory creates a directory named path, along with any necessary parents. +func CreateDirectory(path string) error { + return os.MkdirAll(path, 0755) +} + +// IsPathExist returns true if the path(file or directory) exist. +func IsPathExist(path string) bool { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return false + } + } + return true +} + +// IsDir returns true if the path is an exist directory. +func IsDir(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return false + } + return fi.IsDir() +} + +// IsNotDir returns true if the path is not a directory(not exist or not a directory). +func IsNotDir(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return true + } + return !fi.IsDir() +} + +// IsRegular returns true if the path is an exist regular file. +func IsRegular(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return false + } + return fi.Mode().IsRegular() +} + +// IsNotRegular returns true if the path is not a regular file(not exist or not a regular file). +func IsNotRegular(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return true + } + return !fi.Mode().IsRegular() +} + +// IsSymlink returns true if the path is a symbolic link. +func IsSymlink(path string) bool { + fi, err := os.Lstat(path) + if err != nil { + return false + } + return (fi.Mode() & fs.ModeSymlink) != 0 +} + +// IsNotSymlink returns true if the path is not a symbolic link(not exist or not a symbolic link). +func IsNotSymlink(path string) bool { + fi, err := os.Lstat(path) + if err != nil { + return true + } + return (fi.Mode() & fs.ModeSymlink) == 0 +} + +// SearchFile returns an array of filepath that match. +func SearchFile(root string, pattern string) []string { + result := make([]string, 0) + re := regexp.MustCompile(pattern) + filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsRegular() && re.MatchString(info.Name()) { + result = append(result, path) + } + return nil + }) + + return result +} diff --git a/utils/osutil/syscall.go b/utils/osutil/syscall.go new file mode 100644 index 0000000..3764463 --- /dev/null +++ b/utils/osutil/syscall.go @@ -0,0 +1,39 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package osutil + +import ( + "fmt" + "math" + + "golang.org/x/sys/unix" +) + +// DiskUsage estimate file space usage, returns the command struct and exec status at last. +func DiskUsage(path string) (map[string]interface{}, error) { + result := make(map[string]interface{}) + var buf unix.Statfs_t + if err := unix.Statfs(path, &buf); err != nil { + return result, err + } + + total := buf.Blocks * uint64(buf.Bsize) + free := buf.Bfree * uint64(buf.Bsize) + result["total"] = total + result["free"] = free + return result, nil +} + +// FormatStorage formats storage from a bytes to a human-readable string. +func FormatStorage(bytes uint64) string { + if bytes == 0 { + return "0 B" + } + units := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} + n := math.Floor(math.Log(float64(bytes)) / math.Log(1024)) + i := int(n) + if 0 <= i && len(units) > i { + return fmt.Sprintf("%.2f %s", float64(bytes)/math.Pow(1024, n), units[i]) + } + return "nil" +} diff --git a/utils/strutil/encoding.go b/utils/strutil/encoding.go new file mode 100644 index 0000000..1358f96 --- /dev/null +++ b/utils/strutil/encoding.go @@ -0,0 +1,33 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package strutil + +import ( + "encoding/base64" + "fmt" + "log" + + "github.com/spaolacci/murmur3" +) + +// Base64URLEncode returns a base64 encoded string of v. +func Base64URLEncode(v string) string { + return base64.RawURLEncoding.EncodeToString([]byte(v)) +} + +// Base64URLDecode returns a decoded string with base64 of v. +func Base64URLDecode(v string) string { + buf, err := base64.RawURLEncoding.DecodeString(v) + if err != nil { + log.Println(err) + } + return string(buf) +} + +// Base64URLHash returns a base64 encoded string hashed with murmur of v. +func Base64URLHash(v interface{}) string { + str := fmt.Sprint(v) + sum := murmur3.Sum64([]byte(str)) + buf := fmt.Sprintf("%.16x", sum) + return Base64URLEncode(buf) +} diff --git a/utils/strutil/escape.go b/utils/strutil/escape.go new file mode 100644 index 0000000..41eb28d --- /dev/null +++ b/utils/strutil/escape.go @@ -0,0 +1,80 @@ +// Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +package strutil + +import ( + "bytes" +) + +var JSONHTMLRules = []map[string]string{ + { + "\\\"": "\"", + }, + { + "\\\"": "\"", + "\\\\n": "\\n", + }, + { + "\\\\u0026": "&", + "\\\\u003c": "<", + "\\\\u003e": ">", + }, +} + +var HTMLRules = map[string]string{ + "\"": """, + "&": "&", + "<": "<", + ">": ">", +} + +var XMLRules = map[string]string{ + "'": "'", + "\"": """, + "&": "&", + "<": "<", + ">": ">", +} + +// ReplaceAll returns a copy of the slice s with the rules that: replace name with value if reverse is false, else replae value with name. +func ReplaceAll(s []byte, rules map[string]string, reverse bool) []byte { + if !reverse { + for name, value := range rules { + s = bytes.ReplaceAll(s, []byte(name), []byte(value)) + } + } else { + for name, value := range rules { + s = bytes.ReplaceAll(s, []byte(value), []byte(name)) + } + } + + return s +} + +// JSONHTMLUnescape returns a copy of the slice s with JSONHTMLRules. +func JSONHTMLUnescape(s []byte) []byte { + for i := 0; i < len(JSONHTMLRules); i++ { + s = ReplaceAll(s, JSONHTMLRules[i], false) + } + return s +} + +// HTMLEscape returns a copy of the slice s with HTMLRules. +func HTMLEscape(s []byte) []byte { + return ReplaceAll(s, HTMLRules, false) +} + +// HTMLUnescape returns a copy of the slice s with HTMLRules. +func HTMLUnescape(s []byte) []byte { + return ReplaceAll(s, HTMLRules, true) +} + +// XMLEscape returns a copy of the slice s with XMLRules. +func XMLEscape(s []byte) []byte { + return ReplaceAll(s, XMLRules, false) +} + +// XMLUnescape returns a copy of the slice s with XMLRules. +func XMLUnescape(s []byte) []byte { + return ReplaceAll(s, XMLRules, true) +} diff --git a/utils/strutil/strings.go b/utils/strutil/strings.go new file mode 100644 index 0000000..e7b7f6b --- /dev/null +++ b/utils/strutil/strings.go @@ -0,0 +1,195 @@ +// Copyright 2010-2023 nanzoom.com. All Rights Reserved. + +package strutil + +import ( + "math" + "strings" +) + +// Index returns the index of the first instance of substr in s, or -1 if substr is not present in s. +func Index(s, substr string) int { + r, subrunes := []rune(s), []rune(substr) + rlen, subrlen := len(r), len(subrunes) + for i := 0; i <= rlen-subrlen; i++ { + if string(r[i:i+subrlen]) == substr { + return i + } + } + return -1 +} + +// Indexof returns the index of the first instance of substr in s[i:], or -1 if substr is not present in s[i:]. +func Indexof(s, substr string, i int) int { + if i > len(s)-1 { + return -1 + } + if i <= 0 { + return strings.Index(s, substr) + } + ind := strings.Index(s[i:], substr) + if ind == -1 { + return -1 + } + return ind + i +} + +// LastIndexof returns the index of the last instance of substr in s[i:], or -1 if substr is not present in s[i:]. +func LastIndexof(s, substr string, i int) int { + if i < 0 { + return -1 + } + if i >= len(s) { + return strings.LastIndex(s, substr) + } + ind := strings.LastIndex(s[i:], substr) + if ind == -1 { + return -1 + } + return ind + 1 +} + +// IndexofRune returns the index of the first instance of substr in s[i:], or -1 if substr is not present in s[i:]. +func IndexofRune(s, substr []rune, i int) int { + if i > len(s)-1 { + return -1 + } + if i <= 0 { + return IndexRune(s, substr) + } + ind := IndexRune(s[i:], substr) + if ind == -1 { + return -1 + } + return ind + i +} + +// LastIndexofRune returns the index of the last instance of substr in s[i:], or -1 if substr is not present in s[i:]. +func LastIndexofRune(s, substr []rune, i int) int { + if i < 0 { + return -1 + } + if i >= len(s) { + return LastIndexRune(s, substr) + } + ind := LastIndexRune(s[i:], substr) + if ind == -1 { + return -1 + } + return ind + 1 +} + +// CompareRune is the rune version of strings.Compare. +func CompareRune(a, b []rune) int { + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return 1 + } + + for i, c := range a { + if c < b[i] { + return -1 + } + if c > b[i] { + return 1 + } + } + + return 0 +} + +// IndexRune is the rune version of strings.Index. +func IndexRune(s, substr []rune) int { + need := len(substr) + last := len(s) - need + for i := 0; i <= last; i++ { + if 0 == CompareRune(s[i:i+need], substr) { + return i + } + } + return -1 +} + +// LastIndexRune is the rune version of strings.LastIndex. +func LastIndexRune(s, substr []rune) int { + need := len(substr) + last := len(s) - need + for i := last; i >= 0; i-- { + if 0 == CompareRune(s[i:i+need], substr) { + return i + } + } + return -1 +} + +// CommonLength returns equal length of two texts, +// which start from head when prefix is true, else from tail. +func CommonLength(a, b string, prefix bool) int { + return CommonLengthRune([]rune(a), []rune(b), prefix) +} + +// CommonLengthRune returns equal length of two runes, +// which start from head when prefix is true, else from tail. +func CommonLengthRune(a, b []rune, prefix bool) int { + if prefix { + n := 0 + for ; n < len(a) && n < len(b); n++ { + if a[n] != b[n] { + return n + } + } + return n + } else { + i1 := len(a) + i2 := len(b) + for n := 0; ; n++ { + i1-- + i2-- + if i1 < 0 || i2 < 0 || a[i1] != b[i2] { + return n + } + } + } +} + +// CommonOverlap determines if the suffix of one string is the prefix of another. +func CommonOverlap(text1 string, text2 string) int { + // Cache the text lengths to prevent multiple calls. + text1Length := len(text1) + text2Length := len(text2) + // Eliminate the null case. + if text1Length == 0 || text2Length == 0 { + return 0 + } + // Truncate the longer string. + if text1Length > text2Length { + text1 = text1[text1Length-text2Length:] + } else if text1Length < text2Length { + text2 = text2[0:text1Length] + } + textLength := int(math.Min(float64(text1Length), float64(text2Length))) + // Quick check for the worst case. + if text1 == text2 { + return textLength + } + + // Start by looking for a single character match and increase length until no match is found. Performance analysis: http://neil.fraser.name/news/2010/11/04/ + best := 0 + length := 1 + for { + pattern := text1[textLength-length:] + found := strings.Index(text2, pattern) + if found == -1 { + break + } + length += found + if found == 0 || text1[textLength-length:] == text2[0:length] { + best = length + length++ + } + } + + return best +} diff --git a/utils/utils.sh b/utils/utils.sh new file mode 100644 index 0000000..cb12b6e --- /dev/null +++ b/utils/utils.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Author: bruce(bruce@nanzoom.com) +# Copyright 2010-2022 nanzoom.com. All Rights Reserved. + +DEBUG="yes" +SHELL=`basename $0` +CURDIR=`pwd` + +#---------------------------------------------------------- +# generateSbs(path, output): +# generate code for creating Sbs. +# paremeter: +# path: generate from which, default: ../service/struct.go +# output: output filename, default: stdout +#---------------------------------------------------------- +function generateSbs() { + # parse parameters. + path="$CURDIR/service/struct.go"; if [ $# -ge 1 ]; then + path=$1 + fi + output=/dev/stdout; if [ $# -ge 2 ]; then + output=$2 + fi + if [ $DEBUG == "yes" ]; then + echo "input($*), output(path:$path,output:$output)" + fi + + # prepare variables. + + # execute. + IFS=$'\n' + echo "map[string]*sqlbuilder.Struct{" >$output + for line in `cat $path |grep -E "^type\s+\w+\s+struct\s+{\w*$"`; do + name=`echo $line |awk '{print $2}'` + echo " \"$name\": sqlbuilder.NewStruct(new($name)).For(sqlbuilder.PostgreSQL)," >>$output + done + echo "}" >>$output +} + +#---------------------------------------------------------- +# generateStruct(dbname, username, output): +# generate code for table structures. +# paremeter: +# dbname: database name to connect to. +# username: database user name. +# output: output filename. +#---------------------------------------------------------- +function generateStruct() { + # parse parameters. + dbname="serverdb"; if [ $# -ge 1 ]; then + dbname=$1 + fi + username="postgres"; if [ $# -ge 2 ]; then + username=$2 + fi + output="struct.txt"; if [ $# -ge 3 ]; then + output=$3 + fi + if [ $DEBUG == "yes" ]; then + echo "input($*), output(dbname:$dbname,username:$username,output:$output)" + fi + + # prepare variables. + sqlfile="$CURDIR/db.sql" + + # execute. + IFS=$'\n' + echo -e "// this file is created by utils.sh.generateStruct.\n\n" >$output + echo "\t" >$sqlfile; echo "\dt" >>$sqlfile + for name in `psql -U $username -d $dbname -f $sqlfile |awk -F "|" '{print $2}'`; do + name=`echo $name |sed 's/^[ \t]*//g' |sed 's/[ \t]*$//g'` + exportTableDict $name + done + rm -f $sqlfile +} + +function exportTableDict() { + local name=$1 + local sqlfile="$CURDIR/temp.sql" + local tmpfile="$CURDIR/temp.txt" + + pascal=`echo $name |sed --expression 's/_\([a-z]\)/\U\1/g' --expression 's/^\(.\)/\U\1/g'` + echo "// $pascal is table struct." >>$output + echo "type $pascal struct {" >>$output + echo "\t" >$sqlfile; echo "\d $name" >>$sqlfile + for row in `psql -U $username -d $dbname -f $sqlfile |grep -v "Tuples only"`; do + column=`echo $row |awk -F "|" '{print $1}' |sed 's/^[ \t]*//g' |sed 's/[ \t]*$//g'` + type=`echo $row |awk -F "|" '{print $2}' |sed 's/^[ \t]*//g' |sed 's/[ \t]*$//g'` + nullable=`echo $row |awk -F "|" '{print $4}' |sed 's/^[ \t]*//g' |sed 's/[ \t]*$//g'`; if [ "$nullable"=="not null" ]; then + nullable="Y" + fi + default=`echo $row |awk -F "|" '{print $5}' |sed 's/^[ \t]*//g' |sed 's/[ \t]*$//g'` + pascal=`echo $column |sed --expression 's/_\([a-z]\)/\U\1/g' --expression 's/^\(.\)/\U\1/g'` + typestr="*string" + istime=`echo $type |grep "timestamp"`; if [ -n "$istime" ]; then + typestr="*time.Time" + fi + isserial=`echo $default |grep -e "^nextval"` + + if [ -n "$isserial" ]; then + echo " $pascal $typestr \`db:\"$column\" fieldopt:\"omitempty\"\`" >>$output + else + echo " $pascal $typestr \`db:\"$column\"\`" >>$output + fi + done + echo -e "}\n" >>$output + rm -f $sqlfile $tmpfile +} + +# generateSbs "$@" +generateStruct "$@" \ No newline at end of file diff --git a/uzsdk/application.go b/uzsdk/application.go index 5c3dd03..ca0e2c2 100644 --- a/uzsdk/application.go +++ b/uzsdk/application.go @@ -1,10 +1,11 @@ -// Copyright 2023 nanzoom.com. All Rights Reserved. +// Copyright 2010-2024 nanzoom.com. All Rights Reserved. package uzsdk // Application defines structure of application's information. type Application struct { NAME string + UUID string VENDOR string AUTHOR string VERSION string diff --git a/uzsdk/configuration.go b/uzsdk/configuration.go index fff9646..e4609a8 100644 --- a/uzsdk/configuration.go +++ b/uzsdk/configuration.go @@ -1,4 +1,4 @@ -// Copyright 2010-2022 nanzoom.com. All Rights Reserved. +// Copyright 2010-2024 nanzoom.com. All Rights Reserved. package uzsdk @@ -14,4 +14,31 @@ type Configuration struct { DataDirectory string // server data directory, default: /data/cloud TrustedProxies []string // a list of proxies trusted, default: ["127.0.0.1"] } + System struct { + DBDriver string // database driver, default: postgres + DBHost string // database host, default: 127.0.0.1 + DBPort int64 // database port, default: 5432 + DBUser string // database user, default: postgres + DBPassword string // database password, default: postgres + DBName string // database name, default: serverdb + Heartbeat int64 // heartbeat(unit: second) for keepalive between client and server, default: 36000 + IdleTime int64 // server kick off client those do nothing after this idle time, default: [Heartbeat]<<2 + CheckTime int64 // interval that server check client status, default: [Heartbeat]>>1 + DbgOutput bool // debug: output more details, default: fasle. + DbgToken bool // debug: use token '63886e35-7291-4b4d-a66f-fa7431542ace', default: false + } + GZip struct { + Enable bool // enable gzip with gin, default: true + Level int64 // gzip compress level, default: gzip.DefaultCompression + } + Logging struct { + Level string // TODO: log level, default: error + Prefix string // prefix add to each log message, default: "" + File struct { + Name string // log filename + } + Syslog struct { + Enable bool // TODO: enable log to system, default: false + } + } } diff --git a/uzsdk/service.go b/uzsdk/service.go index ac63db6..42dfd05 100644 --- a/uzsdk/service.go +++ b/uzsdk/service.go @@ -1,4 +1,4 @@ -// Copyright 2023 nanzoom.com. All Rights Reserved. +// Copyright 2010-2024 nanzoom.com. All Rights Reserved. package uzsdk @@ -140,6 +140,7 @@ type TokenCheckFunc func(token string, options ...interface{}) bool // Instance is the framework's instance, used for server & plugin both. type Instance struct { + Application *Application // application of this application. Conf *Configuration // configuration of this application. Db *sqlx.DB // the first database connection created by main routine. Logger *logger.Logger // logger created by main routine. diff --git a/version.go b/version.go index 61e98f8..da246ac 100644 --- a/version.go +++ b/version.go @@ -1,69 +1,5 @@ -// Copyright 2023 nanzoom.com. All Rights Reserved. -/* ## Package versions -go 1.20 - -require ( - git.uzoombox.com/git/uzsdk v0.1.0 - github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/chromedp/cdproto v0.0.0-20230611221135-4cd95c996604 - github.com/chromedp/chromedp v0.9.1 - github.com/cstockton/go-conv v1.0.0 - github.com/fatih/structs v1.1.0 - github.com/gin-contrib/gzip v0.0.6 - github.com/gin-gonic/gin v1.9.1 - github.com/huandu/go-sqlbuilder v1.21.0 - github.com/jmoiron/sqlx v1.3.5 - github.com/json-iterator/go v1.1.12 - github.com/lib/pq v1.10.9 - github.com/mafredri/cdp v0.34.1 - github.com/pdfcpu/pdfcpu v0.4.1 - github.com/spaolacci/murmur3 v1.1.0 - github.com/wechatpay-apiv3/wechatpay-go v0.2.16 - golang.org/x/sys v0.8.0 -) - -require ( - github.com/bytedance/sonic v1.9.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/chromedp/sysutil v1.0.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect - github.com/gobwas/httphead v0.1.0 // indirect - github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.1.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/hhrutter/lzw v1.0.0 // indirect - github.com/hhrutter/tiff v1.0.0 // indirect - github.com/huandu/xstrings v1.3.2 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) -*/ - +// Copyright 2010-2024 nanzoom.com. All Rights Reserved. package uzsdk // Version is the current opensdk's version. -const Version = "0.2.5" +const Version = "0.3.0" -- 2.43.5