├── .gitignore ├── bin └── .gitignore ├── go.mod ├── .dockerignore ├── .github └── workflows │ └── release.yml ├── Dockerfile ├── Makefile ├── go.sum ├── LICENSE.txt ├── utils └── generator │ └── main.go ├── redisdump ├── tls_utils.go ├── redisdump_test.go └── redisdump.go ├── Readme.md └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | bin 3 | .idea/ 4 | -------------------------------------------------------------------------------- /bin/.gitignore: -------------------------------------------------------------------------------- 1 | upstash-redis-dump 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/upstash/upstash-redis-dump 2 | 3 | go 1.16 4 | 5 | require github.com/mediocregopher/radix/v3 v3.8.0 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Ignore unnecessary files during Docker build 2 | .git 3 | .gitignore 4 | README.md 5 | Readme.md 6 | LICENSE.txt 7 | Makefile 8 | 9 | # Ignore build artifacts 10 | bin/ 11 | dist/ 12 | *.zip 13 | 14 | # Ignore test files and directories 15 | test/ 16 | *_test.go 17 | 18 | # Ignore development utilities 19 | utils/ 20 | 21 | # Ignore IDE and editor files 22 | .vscode/ 23 | .idea/ 24 | *.swp 25 | *.swo 26 | *~ 27 | 28 | # Ignore OS files 29 | .DS_Store 30 | Thumbs.db 31 | 32 | # Ignore Docker files themselves (except Dockerfile) 33 | .dockerignore 34 | docker-compose.yml 35 | docker-compose.yaml 36 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | name: Create Release 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v2 16 | 17 | - name: Setup Go 18 | uses: actions/setup-go@v2 19 | with: 20 | stable: 'true' 21 | go-version: '1.17' 22 | 23 | - name: Build 24 | run: make release 25 | 26 | - name: Release 27 | uses: softprops/action-gh-release@v1 28 | with: 29 | token: ${{ secrets.GITHUB_TOKEN }} 30 | files: dist/* 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-alpine AS builder 2 | 3 | RUN apk add --no-cache git ca-certificates 4 | WORKDIR /app 5 | COPY go.mod go.sum ./ 6 | RUN go mod download 7 | COPY . . 8 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o upstash-redis-dump . 9 | 10 | FROM alpine:latest 11 | 12 | RUN apk --no-cache add ca-certificates redis 13 | RUN addgroup -g 1001 -S appuser && \ 14 | adduser -u 1001 -S appuser -G appuser 15 | 16 | WORKDIR /app 17 | COPY --from=builder /app/upstash-redis-dump . 18 | RUN chown appuser:appuser upstash-redis-dump 19 | 20 | RUN ln -s /app/upstash-redis-dump /bin/upstash-redis-dump 21 | 22 | USER appuser 23 | 24 | ENTRYPOINT ["./upstash-redis-dump"] 25 | CMD ["-h"] 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make 2 | 3 | .PHONY: test build 4 | 5 | all: test build 6 | 7 | test: 8 | go test ./... 9 | go vet ./... 10 | 11 | build: 12 | go build -o bin/upstash-redis-dump 13 | 14 | release: 15 | mkdir -p dist 16 | GOOS=linux GOARCH=amd64 go build && zip -m dist/upstash-redis-dump_linux_amd64.zip upstash-redis-dump 17 | GOOS=linux GOARCH=arm64 go build && zip -m dist/upstash-redis-dump_linux_arm64.zip upstash-redis-dump 18 | GOOS=darwin GOARCH=amd64 go build && zip -m dist/upstash-redis-dump_macos_amd64.zip upstash-redis-dump 19 | GOOS=darwin GOARCH=arm64 go build && zip -m dist/upstash-redis-dump_macos_arm64.zip upstash-redis-dump 20 | GOOS=windows GOARCH=amd64 go build && zip -m dist/upstash-redis-dump_windows_amd64.zip upstash-redis-dump.exe 21 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/mediocregopher/radix/v3 v3.8.0 h1:HI8EgkaM7WzsrFpYAkOXIgUKbjNonb2Ne7K6Le61Pmg= 4 | github.com/mediocregopher/radix/v3 v3.8.0/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 8 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 9 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= 10 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 11 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 17 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /utils/generator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "math/rand" 8 | "os" 9 | "strings" 10 | 11 | "github.com/upstash/upstash-redis-dump/redisdump" 12 | ) 13 | 14 | var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 15 | 16 | func randSeq(n int) string { 17 | b := make([]rune, n) 18 | for i := range b { 19 | b[i] = letters[rand.Intn(len(letters))] 20 | } 21 | return string(b) 22 | } 23 | 24 | func GenerateStrings(w io.Writer, nKeys int, serializer redisdump.Serializer) { 25 | for i := 0; i < nKeys; i++ { 26 | io.WriteString(w, serializer([]string{"SET", randSeq(8), randSeq(16)})+"\n") 27 | } 28 | } 29 | 30 | func GenerateZSET(w io.Writer, nKeys int, serializer redisdump.Serializer) { 31 | zsetKey := randSeq(16) 32 | for i := 0; i < nKeys; i++ { 33 | io.WriteString(w, serializer([]string{"ZADD", zsetKey, "1", randSeq(16)})+"\n") 34 | } 35 | } 36 | 37 | func main() { 38 | nKeys := flag.Int("n", 100, "Number of keys to generate") 39 | sType := flag.String("type", "strings", "zset or strings") 40 | oType := flag.String("output", "resp", "resp or commands") 41 | flag.Parse() 42 | 43 | var s redisdump.Serializer 44 | switch strings.ToLower(*oType) { 45 | case "resp": 46 | s = redisdump.RESPSerializer 47 | 48 | case "commands": 49 | s = redisdump.RedisCmdSerializer 50 | 51 | default: 52 | fmt.Fprintf(os.Stderr, "Unrecognised type %s, should be strings or zset", *sType) 53 | os.Exit(1) 54 | } 55 | 56 | switch strings.ToLower(*sType) { 57 | case "zset": 58 | GenerateZSET(os.Stdout, *nKeys, s) 59 | 60 | case "strings": 61 | GenerateStrings(os.Stdout, *nKeys, s) 62 | 63 | default: 64 | fmt.Fprintf(os.Stderr, "Unrecognised type %s, should be strings or zset", *sType) 65 | os.Exit(1) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /redisdump/tls_utils.go: -------------------------------------------------------------------------------- 1 | package redisdump 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "io/ioutil" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/mediocregopher/radix/v3" 12 | ) 13 | 14 | type TlsHandler struct { 15 | tls bool 16 | caCertPath string 17 | certPath string 18 | keyPath string 19 | } 20 | 21 | func NewTlsHandler(tls bool, caCertPath, certPath, keyPath string) *TlsHandler { 22 | return &TlsHandler{ 23 | tls: tls, 24 | caCertPath: caCertPath, 25 | certPath: certPath, 26 | keyPath: keyPath, 27 | } 28 | } 29 | 30 | func NewRedisClient(redisURL string, tlsHandler *TlsHandler, redisPassword string, nWorkers int, db string) (*radix.Pool, error) { 31 | tlsConfig, err := createTlsConfig(tlsHandler) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | customConnFunc := func(network, addr string) (radix.Conn, error) { 37 | return newRedisConn(network, addr, redisPassword, tlsConfig, db) 38 | } 39 | return radix.NewPool("tcp", redisURL, nWorkers, radix.PoolConnFunc(customConnFunc)) 40 | } 41 | 42 | func NewRedisConn(redisURL string, tlsHandler *TlsHandler, redisPassword string, db string) (radix.Conn, error) { 43 | tlsConfig, err := createTlsConfig(tlsHandler) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return newRedisConn("tcp", redisURL, redisPassword, tlsConfig, db) 48 | } 49 | 50 | func createTlsConfig(tlsHandler *TlsHandler) (*tls.Config, error) { 51 | var tlsConfig *tls.Config 52 | if tlsHandler != nil { 53 | // ca cert is optional 54 | var certPool *x509.CertPool 55 | if tlsHandler.caCertPath != "" { 56 | pem, err := ioutil.ReadFile(tlsHandler.caCertPath) 57 | if err != nil { 58 | return nil, fmt.Errorf("connectionpool: unable to open CA certs: %v", err) 59 | } 60 | 61 | certPool = x509.NewCertPool() 62 | if !certPool.AppendCertsFromPEM(pem) { 63 | return nil, fmt.Errorf("connectionpool: failed parsing or CA certs") 64 | } 65 | } 66 | tlsConfig = &tls.Config{ 67 | Certificates: []tls.Certificate{}, 68 | RootCAs: certPool, 69 | } 70 | if tlsHandler.certPath != "" && tlsHandler.keyPath != "" { 71 | cert, err := tls.LoadX509KeyPair(tlsHandler.certPath, tlsHandler.keyPath) 72 | if err != nil { 73 | return nil, err 74 | } 75 | tlsConfig.Certificates = append(tlsConfig.Certificates, cert) 76 | } 77 | } 78 | return tlsConfig, nil 79 | } 80 | 81 | func newRedisConn(network, redisURL string, redisPassword string, tlsConfig *tls.Config, db string) (radix.Conn, error) { 82 | dialOpts := []radix.DialOpt{ 83 | radix.DialTimeout(5 * time.Minute), 84 | } 85 | if redisPassword != "" { 86 | dialOpts = append(dialOpts, radix.DialAuthPass(redisPassword)) 87 | } 88 | if tlsConfig != nil { 89 | dialOpts = append(dialOpts, radix.DialUseTLS(tlsConfig)) 90 | } 91 | if db != "" { 92 | dbVal, err := strconv.Atoi(db) 93 | if err != nil { 94 | return nil, err 95 | } 96 | dialOpts = append(dialOpts, radix.DialSelectDB(dbVal)) 97 | } 98 | return radix.Dial(network, redisURL, dialOpts...) 99 | } 100 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Redis Dump 2 | 3 | **Note:** _This project is started as a fork of Yann Hamon's [redis-dump-go](https://github.com/yannh/redis-dump-go)._ 4 | 5 | ___ 6 | 7 | > [!NOTE] 8 | > **This project is in the Experimental Stage.** 9 | > 10 | > We declare this project experimental to set clear expectations for your usage. There could be known or unknown bugs, the API could evolve, or the project could be discontinued if it does not find community adoption. While we cannot provide professional support for experimental projects, we’d be happy to hear from you if you see value in this project! 11 | 12 | Dumps Redis keys & values to a file. Similar in spirit to https://www.npmjs.com/package/redis-dump and https://github.com/delano/redis-dump but: 13 | 14 | * Will dump keys across **several processes & connections** 15 | * Uses SCAN rather than KEYS * for much **reduced memory footprint** with large databases 16 | * Easy to deploy & containerize - **single binary**. 17 | * Generates a [RESP](https://redis.io/topics/protocol) file rather than a JSON or a list of commands. This is **faster to ingest**, and [recommended by Redis](https://redis.io/topics/mass-insert) for mass-inserts. 18 | 19 | ## Features 20 | 21 | * Dumps all databases present on the Redis server 22 | * Keys TTL are preserved by default 23 | * Configurable Output (Redis commands, RESP) 24 | * Redis password-authentication 25 | > [!NOTE] 26 | > This tool does not dump Redis Stream entries. 27 | 28 | 29 | ## Installation 30 | 31 | You can download one of the pre-built binaries for Linux, Windows and macOS from the 32 | [releases](https://github.com/upstash/upstash-redis-dump/releases/latest) page. 33 | 34 | Or if you have Go SDK installed on your system, you can get the latest `upstash-redis-dump` by running: 35 | 36 | ```bash 37 | go install github.com/upstash/upstash-redis-dump@latest 38 | ``` 39 | 40 | ## Run 41 | 42 | ``` 43 | $ upstash-redis-dump -h 44 | Usage of upstash-redis-dump: 45 | -batchSize int 46 | HSET/RPUSH/SADD/ZADD only add 'batchSize' items at a time (default 1000) 47 | -cacert string 48 | TLS CACert file path 49 | -cert string 50 | TLS Cert file path 51 | -db uint 52 | only dump this database (default: all databases) 53 | -filter string 54 | Key filter to use (default "*") 55 | -host string 56 | Server host (default "127.0.0.1") 57 | -key string 58 | TLS Key file path 59 | -n int 60 | Parallel workers (default 10) 61 | -noscan 62 | Use KEYS * instead of SCAN - for Redis <=2.8 63 | -output string 64 | Output type - can be resp or commands (default "resp") 65 | -pass string 66 | Server password 67 | -port int 68 | Server port (default 6379) 69 | -s Silent mode (disable logging of progress / stats) 70 | -tls 71 | Enable TLS 72 | -ttl 73 | Preserve Keys TTL (default true) 74 | 75 | ``` 76 | 77 | ## Sample Export 78 | 79 | ```bash 80 | $ upstash-redis-dump -db 0 -host eu1-moving-loon-6379.upstash.io -port 6379 -pass PASSWORD -tls > redis.dump 81 | Database 0: 9 keys dumped 82 | ``` 83 | 84 | ## Importing the data 85 | 86 | ``` 87 | redis-cli --tls -u redis://REDIS_PASSWORD@gusc1-cosmic-heron-6379.upstash.io:6379 --pipe < redis.dump 88 | ``` 89 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/url" 9 | "os" 10 | "sync" 11 | 12 | "github.com/upstash/upstash-redis-dump/redisdump" 13 | ) 14 | 15 | type progressLogger struct { 16 | stats map[uint8]int 17 | } 18 | 19 | func newProgressLogger() *progressLogger { 20 | return &progressLogger{ 21 | stats: map[uint8]int{}, 22 | } 23 | } 24 | 25 | func (p *progressLogger) drawProgress(to io.Writer, db uint8, nDumped int) { 26 | if _, ok := p.stats[db]; !ok && len(p.stats) > 0 { 27 | // We switched database, write to a new line 28 | fmt.Fprintf(to, "\n") 29 | } 30 | 31 | p.stats[db] = nDumped 32 | if nDumped == 0 { 33 | return 34 | } 35 | 36 | fmt.Fprintf(to, "\rDatabase %d: %d keys dumped", db, nDumped) 37 | } 38 | 39 | func isFlagPassed(name string) bool { 40 | found := false 41 | flag.Visit(func(f *flag.Flag) { 42 | if f.Name == name { 43 | found = true 44 | } 45 | }) 46 | return found 47 | } 48 | 49 | func realMain() int { 50 | var err error 51 | 52 | host := flag.String("host", "127.0.0.1", "Server host") 53 | port := flag.Int("port", 6379, "Server port") 54 | pass := flag.String("pass", "", "Server password") 55 | db := flag.Uint("db", 0, "only dump this database (default: all databases)") 56 | filter := flag.String("filter", "*", "Key filter to use") 57 | noscan := flag.Bool("noscan", false, "Use KEYS * instead of SCAN - for Redis <=2.8") 58 | batchSize := flag.Int("batchSize", 1000, "HSET/RPUSH/SADD/ZADD only add 'batchSize' items at a time") 59 | nWorkers := flag.Int("n", 10, "Parallel workers") 60 | withTTL := flag.Bool("ttl", true, "Preserve Keys TTL") 61 | output := flag.String("output", "resp", "Output type - can be resp or commands") 62 | silent := flag.Bool("s", false, "Silent mode (disable logging of progress / stats)") 63 | tls := flag.Bool("tls", false, "Enable TLS") 64 | caCert := flag.String("cacert", "", "TLS CACert file path") 65 | cert := flag.String("cert", "", "TLS Cert file path") 66 | key := flag.String("key", "", "TLS Key file path") 67 | flag.Parse() 68 | 69 | if !isFlagPassed("db") { 70 | db = nil 71 | } 72 | var tlshandler *redisdump.TlsHandler 73 | if isFlagPassed("tls") { 74 | *tls = true 75 | tlshandler = redisdump.NewTlsHandler(*tls, *caCert, *cert, *key) 76 | } 77 | 78 | var serializer func([]string) string 79 | switch *output { 80 | case "resp": 81 | serializer = redisdump.RESPSerializer 82 | 83 | case "commands": 84 | serializer = redisdump.RedisCmdSerializer 85 | 86 | default: 87 | log.Fatalf("Failed parsing parameter flag: can only be resp or json") 88 | } 89 | 90 | progressNotifs := make(chan redisdump.ProgressNotification) 91 | var wg sync.WaitGroup 92 | wg.Add(1) 93 | 94 | defer func() { 95 | close(progressNotifs) 96 | wg.Wait() 97 | if !(*silent) { 98 | fmt.Fprint(os.Stderr, "\n") 99 | } 100 | }() 101 | 102 | pl := newProgressLogger() 103 | go func() { 104 | for n := range progressNotifs { 105 | if !(*silent) { 106 | pl.drawProgress(os.Stderr, n.Db, n.Done) 107 | } 108 | } 109 | wg.Done() 110 | }() 111 | logger := log.New(os.Stdout, "", 0) 112 | if db == nil { 113 | if err = redisdump.DumpServer(*host, *port, url.QueryEscape(*pass), tlshandler, *filter, *nWorkers, *withTTL, *batchSize, *noscan, logger, serializer, progressNotifs); err != nil { 114 | fmt.Fprintf(os.Stderr, "%s", err) 115 | return 1 116 | } 117 | } else { 118 | if err = redisdump.DumpDB(*host, *port, url.QueryEscape(*pass), uint8(*db), tlshandler, *filter, *nWorkers, *withTTL, *batchSize, *noscan, logger, serializer, progressNotifs); err != nil { 119 | fmt.Fprintf(os.Stderr, "%s", err) 120 | return 1 121 | } 122 | } 123 | return 0 124 | } 125 | 126 | func main() { 127 | os.Exit(realMain()) 128 | } 129 | -------------------------------------------------------------------------------- /redisdump/redisdump_test.go: -------------------------------------------------------------------------------- 1 | package redisdump 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func testEqString(a, b []string) bool { 8 | 9 | if a == nil && b == nil { 10 | return true 11 | } 12 | 13 | if a == nil || b == nil { 14 | return false 15 | } 16 | 17 | if len(a) != len(b) { 18 | return false 19 | } 20 | 21 | for i := range a { 22 | if a[i] != b[i] { 23 | return false 24 | } 25 | } 26 | 27 | return true 28 | } 29 | 30 | func testEqUint8(a, b []uint8) bool { 31 | // If one is nil, the other must also be nil. 32 | if (a == nil) != (b == nil) { 33 | return false 34 | } 35 | if len(a) != len(b) { 36 | return false 37 | } 38 | for i := range a { 39 | if a[i] != b[i] { 40 | return false 41 | } 42 | } 43 | 44 | return true 45 | } 46 | 47 | func TestStringToRedisCmd(t *testing.T) { 48 | type testCase struct { 49 | key, value string 50 | expected []string 51 | } 52 | 53 | testCases := []testCase{ 54 | {key: "city", value: "Paris", expected: []string{"SET", "city", "Paris"}}, 55 | {key: "fullname", value: "Jean-Paul Sartre", expected: []string{"SET", "fullname", "Jean-Paul Sartre"}}, 56 | {key: "unicode", value: "😈", expected: []string{"SET", "unicode", "😈"}}, 57 | } 58 | 59 | for _, test := range testCases { 60 | res := stringToRedisCmd(test.key, test.value) 61 | if !testEqString(res, test.expected) { 62 | t.Errorf("Failed generating redis command from string for: %s %s", test.key, test.value) 63 | } 64 | } 65 | } 66 | 67 | func TestHashToRedisCmds(t *testing.T) { 68 | type testCase struct { 69 | key string 70 | value map[string]string 71 | cmdMaxLen int 72 | expected [][]string 73 | } 74 | 75 | testCases := []testCase{ 76 | {key: "Paris", value: map[string]string{"country": "France", "weather": "sunny", "poi": "Tour Eiffel"}, cmdMaxLen: 1, expected: [][]string{{"HSET", "Paris", "country", "France"}, {"HSET", "Paris", "weather", "sunny"}, {"HSET", "Paris", "poi", "Tour Eiffel"}}}, 77 | {key: "Paris", value: map[string]string{"country": "France", "weather": "sunny", "poi": "Tour Eiffel"}, cmdMaxLen: 2, expected: [][]string{{"HSET", "Paris", "country", "France", "weather", "sunny"}, {"HSET", "Paris", "poi", "Tour Eiffel"}}}, 78 | {key: "Paris", value: map[string]string{"country": "France", "weather": "sunny", "poi": "Tour Eiffel"}, cmdMaxLen: 3, expected: [][]string{{"HSET", "Paris", "country", "France", "weather", "sunny", "poi", "Tour Eiffel"}}}, 79 | {key: "Paris", value: map[string]string{"country": "France", "weather": "sunny", "poi": "Tour Eiffel"}, cmdMaxLen: 4, expected: [][]string{{"HSET", "Paris", "country", "France", "weather", "sunny", "poi", "Tour Eiffel"}}}, 80 | } 81 | 82 | for _, test := range testCases { 83 | res := hashToRedisCmds(test.key, test.value, test.cmdMaxLen) 84 | for i := 0; i < len(res); i++ { 85 | for j := 2; j < len(res[i]); j += 2 { 86 | found := false 87 | for k := 0; k < len(test.expected); k++ { 88 | for l := 2; l < len(test.expected[k]); l += 2 { 89 | if res[i][j] == test.expected[k][l] && res[i][j+1] == test.expected[k][l+1] { 90 | found = true 91 | } 92 | } 93 | } 94 | 95 | if found == false { 96 | t.Errorf("Failed generating redis command from Hash for: %s %s, got %s", test.key, test.value, res) 97 | } 98 | } 99 | } 100 | } 101 | } 102 | 103 | func TestSetToRedisCmds(t *testing.T) { 104 | type testCase struct { 105 | key string 106 | value []string 107 | cmdMaxLen int 108 | expected [][]string 109 | } 110 | 111 | testCases := []testCase{ 112 | {key: "myset", value: []string{"1", "2", "3"}, cmdMaxLen: 1, expected: [][]string{{"SADD", "myset", "1"}, {"SADD", "myset", "2"}, {"SADD", "myset", "3"}}}, 113 | {key: "myset", value: []string{"1", "2", "3"}, cmdMaxLen: 2, expected: [][]string{{"SADD", "myset", "1", "2"}, {"SADD", "myset", "3"}}}, 114 | {key: "myset", value: []string{"1", "2", "3"}, cmdMaxLen: 3, expected: [][]string{{"SADD", "myset", "1", "2", "3"}}}, 115 | {key: "myset", value: []string{"1", "2", "3"}, cmdMaxLen: 4, expected: [][]string{{"SADD", "myset", "1", "2", "3"}}}, 116 | } 117 | 118 | for _, testCase := range testCases { 119 | res := setToRedisCmds(testCase.key, testCase.value, testCase.cmdMaxLen) 120 | if len(testCase.expected) != len(res) { 121 | t.Errorf("Failed generating redis command from SET for %s %s %d: got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 122 | continue 123 | } 124 | 125 | for i := 0; i < len(testCase.expected); i++ { 126 | if len(testCase.expected[i]) != len(res[i]) { 127 | t.Errorf("Failed generating redis command from SET for %s %s %d: got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 128 | continue 129 | } 130 | for j := 0; j < len(testCase.expected[i]); j++ { 131 | if res[i][j] != testCase.expected[i][j] { 132 | t.Errorf("Failed generating redis command from SET for %s %s %d: got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 133 | } 134 | } 135 | } 136 | } 137 | } 138 | 139 | func TestZsetToRedisCmds(t *testing.T) { 140 | type testCase struct { 141 | key string 142 | value []string 143 | cmdMaxLen int 144 | expected [][]string 145 | } 146 | 147 | testCases := []testCase{ 148 | {key: "todo", value: []string{"task1", "1", "task2", "2", "task3", "3"}, cmdMaxLen: 1, expected: [][]string{{"ZADD", "todo", "1", "task1"}, {"ZADD", "todo", "2", "task2"}, {"ZADD", "todo", "3", "task3"}}}, 149 | {key: "todo", value: []string{"task1", "1", "task2", "2", "task3", "3"}, cmdMaxLen: 2, expected: [][]string{{"ZADD", "todo", "1", "task1", "2", "task2"}, {"ZADD", "todo", "3", "task3"}}}, 150 | {key: "todo", value: []string{"task1", "1", "task2", "2", "task3", "3"}, cmdMaxLen: 3, expected: [][]string{{"ZADD", "todo", "1", "task1", "2", "task2", "3", "task3"}}}, 151 | {key: "todo", value: []string{"task1", "1", "task2", "2", "task3", "3"}, cmdMaxLen: 4, expected: [][]string{{"ZADD", "todo", "1", "task1", "2", "task2", "3", "task3"}}}, 152 | } 153 | 154 | for _, testCase := range testCases { 155 | res := zsetToRedisCmds(testCase.key, testCase.value, testCase.cmdMaxLen) 156 | if len(testCase.expected) != len(res) { 157 | t.Errorf("Failed generating redis command from ZSET for %s %s %d: got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 158 | continue 159 | } 160 | for i := 0; i < len(res); i++ { 161 | if len(testCase.expected[i]) != len(res[i]) { 162 | t.Errorf("Failed generating redis command from ZSET for %s %s %d: got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 163 | continue 164 | } 165 | for j := 2; j < len(res[i]); j += 2 { 166 | found := false 167 | if res[i][j] == testCase.expected[i][j] && res[i][j+1] == testCase.expected[i][j+1] { 168 | found = true 169 | } 170 | 171 | if found == false { 172 | t.Errorf("Failed generating redis command from ZSet for: %s %s %d, got %s", testCase.key, testCase.value, testCase.cmdMaxLen, res) 173 | } 174 | } 175 | } 176 | } 177 | } 178 | 179 | func TestRESPSerializer(t *testing.T) { 180 | type testCase struct { 181 | command []string 182 | expected string 183 | } 184 | 185 | testCases := []testCase{ 186 | {command: []string{"SET", "key", "value"}, expected: "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n"}, 187 | {command: []string{"SET", "key1", "😈"}, expected: "*3\r\n$3\r\nSET\r\n$4\r\nkey1\r\n$4\r\n😈\r\n"}, 188 | } 189 | 190 | for _, test := range testCases { 191 | s := RESPSerializer(test.command) 192 | if s != test.expected { 193 | t.Errorf("Failed serializing command to redis protocol: expected %s, got %s", test.expected, s) 194 | } 195 | } 196 | } 197 | 198 | func TestRedisCmdSerializer(t *testing.T) { 199 | type testCase struct { 200 | command []string 201 | expected string 202 | } 203 | 204 | testCases := []testCase{ 205 | {command: []string{"HELLO"}, expected: "HELLO"}, 206 | {command: []string{"HGETALL", "key"}, expected: "HGETALL key"}, 207 | {command: []string{"SET", "key name 1", "key value 1"}, expected: "SET \"key name 1\" \"key value 1\""}, 208 | {command: []string{"HSET", "key1", "key value 1"}, expected: "HSET key1 \"key value 1\""}, 209 | } 210 | 211 | for _, test := range testCases { 212 | s := RedisCmdSerializer(test.command) 213 | if s != test.expected { 214 | t.Errorf("Failed serializing command to redis protocol: expected %s, got %s", test.expected, s) 215 | } 216 | } 217 | } 218 | 219 | func TestParseKeyspaceInfo(t *testing.T) { 220 | keyspaceInfo := `# Keyspace 221 | db0:keys=2,expires=1,avg_ttl=1009946407050 222 | db2:keys=1,expires=0,avg_ttl=0` 223 | 224 | dbIds, err := parseKeyspaceInfo(keyspaceInfo) 225 | if err != nil { 226 | t.Errorf("Failed parsing keyspaceInfo" + err.Error()) 227 | } 228 | if !testEqUint8(dbIds, []uint8{0, 2}) { 229 | t.Errorf("Failed parsing keyspaceInfo: got %v", dbIds) 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /redisdump/redisdump.go: -------------------------------------------------------------------------------- 1 | package redisdump 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/mediocregopher/radix/v3" 13 | ) 14 | 15 | func ttlToRedisCmd(k string, val int64) []string { 16 | return []string{"EXPIREAT", k, fmt.Sprint(time.Now().Unix() + val)} 17 | } 18 | 19 | func stringToRedisCmd(k, val string) []string { 20 | return []string{"SET", k, val} 21 | } 22 | 23 | func hashToRedisCmds(hashKey string, val map[string]string, batchSize int) [][]string { 24 | cmds := [][]string{} 25 | 26 | cmd := []string{"HSET", hashKey} 27 | n := 0 28 | for k, v := range val { 29 | if n >= batchSize { 30 | n = 0 31 | cmds = append(cmds, cmd) 32 | cmd = []string{"HSET", hashKey} 33 | } 34 | cmd = append(cmd, k, v) 35 | n++ 36 | } 37 | 38 | if n > 0 { 39 | cmds = append(cmds, cmd) 40 | } 41 | 42 | return cmds 43 | } 44 | 45 | func setToRedisCmds(setKey string, val []string, batchSize int) [][]string { 46 | cmds := [][]string{} 47 | cmd := []string{"SADD", setKey} 48 | n := 0 49 | for _, v := range val { 50 | if n >= batchSize { 51 | n = 0 52 | cmds = append(cmds, cmd) 53 | cmd = []string{"SADD", setKey} 54 | } 55 | cmd = append(cmd, v) 56 | n++ 57 | } 58 | 59 | if n > 0 { 60 | cmds = append(cmds, cmd) 61 | } 62 | 63 | return cmds 64 | } 65 | 66 | func listToRedisCmds(listKey string, val []string, batchSize int) [][]string { 67 | cmds := [][]string{} 68 | cmd := []string{"RPUSH", listKey} 69 | n := 0 70 | for _, v := range val { 71 | if n >= batchSize { 72 | n = 0 73 | cmds = append(cmds, cmd) 74 | cmd = []string{"RPUSH", listKey} 75 | } 76 | cmd = append(cmd, v) 77 | n++ 78 | } 79 | 80 | if n > 0 { 81 | cmds = append(cmds, cmd) 82 | } 83 | 84 | return cmds 85 | } 86 | 87 | // We break down large ZSETs to multiple ZADD commands 88 | 89 | func zsetToRedisCmds(zsetKey string, val []string, batchSize int) [][]string { 90 | cmds := [][]string{} 91 | var key string 92 | 93 | cmd := []string{"ZADD", zsetKey} 94 | n := 0 95 | for i, v := range val { 96 | if i%2 == 0 { 97 | key = v 98 | continue 99 | } 100 | 101 | if n >= batchSize { 102 | n = 0 103 | cmds = append(cmds, cmd) 104 | cmd = []string{"ZADD", zsetKey} 105 | } 106 | cmd = append(cmd, v, key) 107 | n++ 108 | } 109 | 110 | if n > 0 { 111 | cmds = append(cmds, cmd) 112 | } 113 | 114 | return cmds 115 | } 116 | 117 | type Serializer func([]string) string 118 | 119 | // RedisCmdSerializer will serialize cmd to a string with redis commands 120 | func RedisCmdSerializer(cmd []string) string { 121 | if len(cmd) == 0 { 122 | return "" 123 | } 124 | 125 | buf := strings.Builder{} 126 | buf.WriteString(fmt.Sprintf("%s", cmd[0])) 127 | for i := 1; i < len(cmd); i++ { 128 | if strings.Contains(cmd[i], " ") { 129 | buf.WriteString(fmt.Sprintf(" \"%s\"", cmd[i])) 130 | } else { 131 | buf.WriteString(fmt.Sprintf(" %s", cmd[i])) 132 | } 133 | } 134 | 135 | return buf.String() 136 | } 137 | 138 | // RESPSerializer will serialize cmd to RESP 139 | func RESPSerializer(cmd []string) string { 140 | buf := strings.Builder{} 141 | buf.WriteString("*" + strconv.Itoa(len(cmd)) + "\r\n") 142 | for _, arg := range cmd { 143 | buf.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n") 144 | } 145 | return buf.String() 146 | } 147 | 148 | func dumpKeys(client radix.Client, keys []string, withTTL bool, batchsize int, logger *log.Logger, serializer Serializer) error { 149 | var err error 150 | var redisCmds [][]string 151 | 152 | for _, key := range keys { 153 | var keyType string 154 | 155 | err = client.Do(radix.Cmd(&keyType, "TYPE", key)) 156 | if err != nil { 157 | return err 158 | } 159 | 160 | switch keyType { 161 | case "string": 162 | var val string 163 | if err = client.Do(radix.Cmd(&val, "GET", key)); err != nil { 164 | return err 165 | } 166 | redisCmds = [][]string{stringToRedisCmd(key, val)} 167 | 168 | case "list": 169 | var val []string 170 | if err = client.Do(radix.Cmd(&val, "LRANGE", key, "0", "-1")); err != nil { 171 | return err 172 | } 173 | redisCmds = listToRedisCmds(key, val, batchsize) 174 | 175 | case "set": 176 | var val []string 177 | if err = client.Do(radix.Cmd(&val, "SMEMBERS", key)); err != nil { 178 | return err 179 | } 180 | redisCmds = setToRedisCmds(key, val, batchsize) 181 | 182 | case "hash": 183 | var val map[string]string 184 | if err = client.Do(radix.Cmd(&val, "HGETALL", key)); err != nil { 185 | return err 186 | } 187 | redisCmds = hashToRedisCmds(key, val, batchsize) 188 | 189 | case "zset": 190 | var val []string 191 | if err = client.Do(radix.Cmd(&val, "ZRANGEBYSCORE", key, "-inf", "+inf", "WITHSCORES")); err != nil { 192 | return err 193 | } 194 | redisCmds = zsetToRedisCmds(key, val, batchsize) 195 | 196 | case "none": 197 | 198 | default: 199 | return fmt.Errorf("Key %s is of unreconized type %s", key, keyType) 200 | } 201 | 202 | for _, redisCmd := range redisCmds { 203 | logger.Print(serializer(redisCmd)) 204 | } 205 | 206 | if withTTL { 207 | var ttl int64 208 | if err = client.Do(radix.Cmd(&ttl, "TTL", key)); err != nil { 209 | return err 210 | } 211 | if ttl > 0 { 212 | cmd := ttlToRedisCmd(key, ttl) 213 | logger.Print(serializer(cmd)) 214 | } 215 | } 216 | } 217 | 218 | return nil 219 | } 220 | 221 | func dumpKeysWorker(client radix.Client, keyBatches <-chan []string, withTTL bool, batchSize int, logger *log.Logger, serializer Serializer, errors chan<- error, done chan<- bool) { 222 | for keyBatch := range keyBatches { 223 | if err := dumpKeys(client, keyBatch, withTTL, batchSize, logger, serializer); err != nil { 224 | errors <- err 225 | } 226 | } 227 | done <- true 228 | } 229 | 230 | // ProgressNotification message indicates the progress in dumping the Redis server, 231 | // and can be used to provide a progress visualisation such as a progress bar. 232 | // Done is the number of items dumped, Total is the total number of items to dump. 233 | type ProgressNotification struct { 234 | Db uint8 235 | Done int 236 | } 237 | 238 | func parseKeyspaceInfo(keyspaceInfo string) ([]uint8, error) { 239 | var dbs []uint8 240 | 241 | scanner := bufio.NewScanner(strings.NewReader(keyspaceInfo)) 242 | 243 | for scanner.Scan() { 244 | line := strings.TrimSpace(scanner.Text()) 245 | 246 | if !strings.HasPrefix(line, "db") { 247 | continue 248 | } 249 | 250 | dbIndexString := line[2:strings.IndexAny(line, ":")] 251 | dbIndex, err := strconv.ParseUint(dbIndexString, 10, 8) 252 | if err != nil { 253 | return nil, err 254 | } 255 | if dbIndex > 16 { 256 | return nil, fmt.Errorf("Error parsing INFO keyspace") 257 | } 258 | 259 | dbs = append(dbs, uint8(dbIndex)) 260 | } 261 | 262 | return dbs, nil 263 | } 264 | 265 | func getDBIndexes(redisURL string, redisPassword string, tlsHandler *TlsHandler) ([]uint8, error) { 266 | 267 | client, err := NewRedisClient(redisURL, tlsHandler, redisPassword, 1, "") 268 | if err != nil { 269 | return nil, err 270 | } 271 | defer client.Close() 272 | 273 | var keyspaceInfo string 274 | if err = client.Do(radix.Cmd(&keyspaceInfo, "INFO", "keyspace")); err != nil { 275 | return nil, err 276 | } 277 | 278 | return parseKeyspaceInfo(keyspaceInfo) 279 | } 280 | 281 | func scanKeys(client radix.Client, db uint8, filter string, keyBatches chan<- []string, progressNotifications chan<- ProgressNotification) error { 282 | keyBatchSize := 100 283 | s := radix.NewScanner(client, radix.ScanOpts{Command: "SCAN", Pattern: filter, Count: keyBatchSize}) 284 | 285 | nProcessed := 0 286 | var key string 287 | var keyBatch []string 288 | for s.Next(&key) { 289 | keyBatch = append(keyBatch, key) 290 | if len(keyBatch) >= keyBatchSize { 291 | nProcessed += len(keyBatch) 292 | keyBatches <- keyBatch 293 | keyBatch = nil 294 | progressNotifications <- ProgressNotification{Db: db, Done: nProcessed} 295 | } 296 | } 297 | 298 | keyBatches <- keyBatch 299 | nProcessed += len(keyBatch) 300 | progressNotifications <- ProgressNotification{Db: db, Done: nProcessed} 301 | 302 | return s.Close() 303 | } 304 | 305 | func min(a, b int) int { 306 | if a <= b { 307 | return a 308 | } 309 | return b 310 | } 311 | 312 | func scanKeysLegacy(client radix.Client, db uint8, filter string, keyBatches chan<- []string, progressNotifications chan<- ProgressNotification) error { 313 | keyBatchSize := 100 314 | var err error 315 | var keys []string 316 | if err = client.Do(radix.Cmd(&keys, "KEYS", filter)); err != nil { 317 | return err 318 | } 319 | 320 | for i := 0; i < len(keys); i += keyBatchSize { 321 | batchEnd := min(i+keyBatchSize, len(keys)) 322 | keyBatches <- keys[i:batchEnd] 323 | if progressNotifications != nil { 324 | progressNotifications <- ProgressNotification{db, i} 325 | } 326 | } 327 | 328 | return nil 329 | } 330 | 331 | // RedisURL builds a connect URL given a Host, port 332 | func RedisURL(redisHost string, redisPort string) string { 333 | return redisHost + ":" + fmt.Sprint(redisPort) 334 | } 335 | 336 | // DumpDB dumps all keys from a single Redis DB 337 | func DumpDB(redisHost string, redisPort int, redisPassword string, db uint8, tlsHandler *TlsHandler, filter string, nWorkers int, withTTL bool, batchSize int, noscan bool, logger *log.Logger, serializer Serializer, progress chan<- ProgressNotification) error { 338 | var err error 339 | 340 | keyGenerator := scanKeys 341 | if noscan { 342 | keyGenerator = scanKeysLegacy 343 | } 344 | 345 | errors := make(chan error) 346 | nErrors := 0 347 | go func() { 348 | for err := range errors { 349 | fmt.Fprintln(os.Stderr, "Error: "+err.Error()) 350 | nErrors++ 351 | } 352 | }() 353 | redisURL := RedisURL(redisHost, fmt.Sprint(redisPort)) 354 | client, err := NewRedisClient(redisURL, tlsHandler, redisPassword, nWorkers, fmt.Sprint(db)) 355 | if err != nil { 356 | return err 357 | } 358 | defer client.Close() 359 | 360 | if err = client.Do(radix.Cmd(nil, "SELECT", fmt.Sprint(db))); err != nil { 361 | return err 362 | } 363 | logger.Printf(serializer([]string{"SELECT", fmt.Sprint(db)})) 364 | 365 | done := make(chan bool) 366 | keyBatches := make(chan []string) 367 | for i := 0; i < nWorkers; i++ { 368 | go dumpKeysWorker(client, keyBatches, withTTL, batchSize, logger, serializer, errors, done) 369 | } 370 | 371 | conn, err := NewRedisConn(redisURL, tlsHandler, redisPassword, fmt.Sprint(db)) 372 | if err != nil { 373 | return err 374 | } 375 | defer conn.Close() 376 | 377 | err = keyGenerator(conn, db, filter, keyBatches, progress) 378 | if err != nil { 379 | return err 380 | } 381 | close(keyBatches) 382 | 383 | for i := 0; i < nWorkers; i++ { 384 | <-done 385 | } 386 | 387 | return nil 388 | } 389 | 390 | // DumpServer dumps all Keys from the redis server given by redisURL, 391 | // to the Logger logger. Progress notification informations 392 | // are regularly sent to the channel progressNotifications 393 | func DumpServer(redisHost string, redisPort int, redisPassword string, tlsHandler *TlsHandler, filter string, nWorkers int, withTTL bool, batchSize int, noscan bool, logger *log.Logger, serializer func([]string) string, progress chan<- ProgressNotification) error { 394 | url := RedisURL(redisHost, fmt.Sprint(redisPort)) 395 | dbs, err := getDBIndexes(url, redisPassword, tlsHandler) 396 | if err != nil { 397 | return err 398 | } 399 | for _, db := range dbs { 400 | if err = DumpDB(redisHost, redisPort, redisPassword, db, tlsHandler, filter, nWorkers, withTTL, batchSize, noscan, logger, serializer, progress); err != nil { 401 | return err 402 | } 403 | } 404 | 405 | return nil 406 | } 407 | --------------------------------------------------------------------------------