├── .gitignore ├── bun.lockb ├── src ├── types.ts ├── utils │ ├── logger.ts │ ├── client.ts │ └── mapper.ts ├── schemas │ ├── meilisearch.ts │ └── algolia.ts ├── index.ts ├── cli │ ├── index.ts │ ├── migrate-from-algolia.ts │ └── migrate-from-meilisearch.ts └── scripts │ ├── meilisearch-data-transfer.ts │ └── algolia-data-tranfer.ts ├── tsup.config.ts ├── LICENSE ├── package.json ├── .github └── workflows │ └── release.yaml ├── tsconfig.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .env 4 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/upstash/search-migrator/master/bun.lockb -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export type UpstashRecord = { 2 | id: string; 3 | content: Record; 4 | metadata: Record; 5 | } -------------------------------------------------------------------------------- /src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk" 2 | 3 | export const logger = { 4 | error(...args: unknown[]) { 5 | console.log(chalk.red(...args)) 6 | }, 7 | warn(...args: unknown[]) { 8 | console.log(chalk.yellow(...args)) 9 | }, 10 | info(...args: unknown[]) { 11 | console.log(chalk.cyan(...args)) 12 | }, 13 | success(...args: unknown[]) { 14 | console.log(chalk.green(...args)) 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup" 2 | 3 | const isDev = process.env.npm_lifecycle_event === "dev" 4 | 5 | export default defineConfig({ 6 | clean: true, 7 | entry: ["src/index.ts"], 8 | format: ["esm"], 9 | minify: !isDev, 10 | target: "esnext", 11 | outDir: "dist", 12 | onSuccess: isDev ? "node dist/index.js" : undefined, 13 | banner: { 14 | js: '#!/usr/bin/env node', 15 | }, 16 | }) 17 | -------------------------------------------------------------------------------- /src/utils/client.ts: -------------------------------------------------------------------------------- 1 | import { searchClient } from '@algolia/client-search'; 2 | import { Search } from "@upstash/search"; 3 | import { Meilisearch } from "meilisearch"; 4 | 5 | // Upstash client creation 6 | export function createUpstashClient(url: string, token: string) { 7 | return new Search({url, token}); 8 | } 9 | 10 | // Algolia client creation 11 | export function createAlgoliaClient(appId: string, apiKey: string) { 12 | return searchClient(appId, apiKey); 13 | } 14 | 15 | // Meilisearch client creation 16 | export function createMeilisearchClient(host: string, apiKey: string) { 17 | return new Meilisearch({host, apiKey}); 18 | } 19 | 20 | -------------------------------------------------------------------------------- /src/schemas/meilisearch.ts: -------------------------------------------------------------------------------- 1 | import { Meilisearch } from "meilisearch"; 2 | 3 | export async function getMeilisearchIndexFields( 4 | client: Meilisearch, 5 | indexName: string 6 | ): Promise { 7 | try { 8 | const index = client.index(indexName); 9 | const documents = await index.getDocuments({ limit: 1 }); 10 | if (!documents.results || documents.results.length === 0 || !documents.results[0]) { 11 | throw new Error("No records found in the index"); 12 | } 13 | const firstRecord = documents.results[0]; 14 | const fields = Object.keys(firstRecord as object).filter( 15 | (key) => !key.startsWith('_') && key !== 'id' 16 | ); 17 | return fields; 18 | } catch (error: unknown) { 19 | if (error instanceof Error) { 20 | throw new Error(`Failed to retrieve index fields: ${error.message}`); 21 | } 22 | throw new Error('Failed to retrieve index fields: Unknown error'); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | // #!/usr/bin/env node 2 | 3 | import { runCli } from "./cli/index.js" 4 | import { logger } from "./utils/logger.js" 5 | import { main as migrateAlgolia } from "./scripts/algolia-data-tranfer.js" 6 | import { main as migrateMeilisearch } from "./scripts/meilisearch-data-transfer.js" 7 | const main = async () => { 8 | const results = await runCli() 9 | if (!results) { 10 | return 11 | } 12 | if ("algoliaClient" in results) { // TODO: Do a proper type checking 13 | await migrateAlgolia(results) 14 | } else if ("meilisearchClient" in results) { 15 | await migrateMeilisearch(results) 16 | } 17 | 18 | process.exit(0) 19 | } 20 | 21 | main().catch((err) => { 22 | logger.error("Aborting migration...") 23 | if (err instanceof Error) { 24 | logger.error(err) 25 | } else { 26 | logger.error( 27 | "An unknown error has occurred. Please open an issue on github with the below:" 28 | ) 29 | console.log(err) 30 | } 31 | process.exit(1) 32 | }) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Upstash 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/utils/mapper.ts: -------------------------------------------------------------------------------- 1 | import { UpstashRecord } from "@/types.js"; 2 | 3 | export function mapAlgoliaToUpstash(hit: any, contentKeys: string[]): UpstashRecord { 4 | const content: Record = {}; 5 | for (const key of contentKeys) { 6 | content[key] = hit[key as keyof typeof hit]; 7 | } 8 | 9 | const metadata: Record = {}; 10 | for (const [key, value] of Object.entries(hit)) { 11 | if (key !== 'objectID' && !contentKeys.includes(key)) { 12 | metadata[key] = value; 13 | } 14 | } 15 | return { id: hit.objectID, content, metadata }; 16 | } 17 | 18 | export function mapMeilisearchToUpstash(hit: any, contentKeys: string[]): UpstashRecord { 19 | const content: Record = {}; 20 | for (const key of contentKeys) { 21 | content[key] = hit[key as keyof typeof hit]; 22 | } 23 | const metadata: Record = {}; 24 | for (const [key, value] of Object.entries(hit)) { 25 | if (key !== 'id' && !contentKeys.includes(key)) { 26 | metadata[key] = value; 27 | } 28 | } 29 | return { id: hit.id, content, metadata }; 30 | } -------------------------------------------------------------------------------- /src/schemas/algolia.ts: -------------------------------------------------------------------------------- 1 | import { SearchClient } from "@algolia/client-search"; 2 | 3 | export async function getAlgoliaIndexFields( 4 | client: SearchClient, 5 | indexName: string 6 | ): Promise { 7 | try { 8 | 9 | const response = await client.search([ 10 | { 11 | indexName, 12 | params: { 13 | hitsPerPage: 1, 14 | query: "", 15 | }, 16 | }, 17 | ]); 18 | 19 | // @ts-ignore - We know the structure of the response from Algolia 20 | const firstResult = response.results[0]; 21 | // @ts-ignore - We know hits will be present in the response 22 | if (!firstResult?.hits?.length) { 23 | throw new Error("No records found in the index"); 24 | } 25 | 26 | // @ts-ignore - We know the structure of hits from Algolia 27 | const firstRecord = firstResult.hits[0]; 28 | const fields = Object.keys(firstRecord).filter( 29 | 30 | (key) => !key.startsWith('_') && key !== 'objectID' 31 | ); 32 | 33 | return fields; 34 | } catch (error: unknown) { 35 | if (error instanceof Error) { 36 | throw new Error(`Failed to retrieve index fields: ${error.message}`); 37 | } 38 | throw new Error('Failed to retrieve index fields: Unknown error'); 39 | } 40 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@upstash/search-migrator", 3 | "version": "0.1.1", 4 | "description": "CLI tool to migrate data to Upstash Search", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/upstash/search-migrator" 8 | }, 9 | "main": "dist/index.js", 10 | "type": "module", 11 | "exports": "./dist/index.js", 12 | "files": [ 13 | "dist", 14 | "package.json" 15 | ], 16 | "bin": { 17 | "@upstash/search-migrator": "./dist/index.js" 18 | }, 19 | "scripts": { 20 | "build": "tsup", 21 | "dev": "tsup --watch", 22 | "start": "node dist/index.js" 23 | }, 24 | "keywords": [ 25 | "upstash", 26 | "search", 27 | "migration", 28 | "cli", 29 | "algolia", 30 | "meilisearch" 31 | ], 32 | "author": "Upstash", 33 | "license": "ISC", 34 | "dependencies": { 35 | "@algolia/client-search": "^5.29.0", 36 | "@clack/prompts": "^0.8.1", 37 | "@upstash/search": "^0.1.2", 38 | "chalk": "^5.3.0", 39 | "commander": "^14.0.0", 40 | "picocolors": "^1.1.1", 41 | "meilisearch": "^0.51.0", 42 | "tsup": "^8.3.5" 43 | }, 44 | "devDependencies": { 45 | "@types/commander": "^2.12.5", 46 | "@types/node": "^22.9.1", 47 | "dotenv": "^16.4.7", 48 | "typescript": "^5.6.3" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | release: 10 | name: Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout Repo 14 | uses: actions/checkout@v3 15 | 16 | - name: Set env 17 | run: echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 18 | 19 | - name: Setup Node 20 | uses: actions/setup-node@v2 21 | with: 22 | node-version: 18 23 | 24 | - name: Set package version 25 | run: echo $(jq --arg v "${{ env.VERSION }}" '(.version) = $v' package.json) > package.json 26 | 27 | - name: Setup Bun 28 | uses: oven-sh/setup-bun@v1 29 | with: 30 | bun-version: latest 31 | 32 | - name: Install dependencies 33 | run: bun install 34 | 35 | - name: Build 36 | run: bun run build 37 | 38 | - name: Add npm token 39 | run: echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" > .npmrc 40 | 41 | - name: Publish release candidate 42 | if: "github.event.release.prerelease" 43 | run: npm publish --access public --tag=canary 44 | 45 | - name: Publish 46 | if: "!github.event.release.prerelease" 47 | run: npm publish --access public 48 | -------------------------------------------------------------------------------- /src/cli/index.ts: -------------------------------------------------------------------------------- 1 | import { Command } from "commander" 2 | import { intro, isCancel, outro, select } from "@clack/prompts" 3 | import color from "picocolors" 4 | import { SearchClient } from "@algolia/client-search" 5 | import { Search } from "@upstash/search" 6 | import { runAlgoliaCli } from "./migrate-from-algolia.js" 7 | import { runMeilisearchCli } from "./migrate-from-meilisearch.js" 8 | import { Meilisearch } from "meilisearch" 9 | 10 | export interface AlgoliaCliResults { 11 | algoliaClient: SearchClient 12 | upstashClient: Search 13 | upstashIndexName: string 14 | algoliaIndexName: string 15 | contentKeys: string[] 16 | } 17 | 18 | export interface MeilisearchCliResults { 19 | meilisearchClient: Meilisearch 20 | upstashClient: Search 21 | upstashIndexName: string 22 | meilisearchIndexName: string 23 | contentKeys: string[] 24 | } 25 | 26 | 27 | export async function runCli(): Promise { 28 | const program = new Command() 29 | program 30 | .option("--upstash-url ", "Upstash URL") 31 | .option("--upstash-token ", "Upstash Token") 32 | .option("--algolia-app-id ", "Algolia App ID") 33 | .option("--algolia-api-key ", "Algolia API Key") 34 | .option("--meilisearch-host ", "Meilisearch Host") 35 | .option("--meilisearch-api-key ", "Meilisearch API Key") 36 | .parse(process.argv) 37 | 38 | const options = program.opts() 39 | 40 | console.clear() 41 | 42 | intro(color.bgCyan(" Upstash Index Migrator ")) 43 | 44 | const provider = 45 | (await select({ 46 | message: "Select your provider", 47 | options: [ 48 | { label: "Algolia", value: "algolia" }, 49 | { label: "Meilisearch", value: "meilisearch" }, 50 | ], 51 | })) 52 | 53 | if (isCancel(provider)) { 54 | outro("Migration cancelled.") 55 | return undefined 56 | } 57 | 58 | if (provider === "algolia") { 59 | return await runAlgoliaCli(options) 60 | } 61 | 62 | if (provider === "meilisearch") { 63 | return await runMeilisearchCli(options) 64 | } 65 | 66 | return undefined; 67 | } 68 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "lib": ["DOM", "DOM.Iterable", "ES2021"], 5 | "module": "Node16", 6 | "moduleResolution": "nodenext", 7 | "resolveJsonModule": true, 8 | "allowJs": true, 9 | "checkJs": true, 10 | "baseUrl": "./", 11 | 12 | /* EMIT RULES */ 13 | "outDir": "./dist", 14 | "noEmit": true, // TSUP takes care of emitting js for us, in a MUCH faster way 15 | "declaration": true, 16 | "declarationMap": true, 17 | "sourceMap": true, 18 | "removeComments": true, 19 | 20 | /* TYPE CHECKING RULES */ 21 | "strict": true, 22 | // "noImplicitAny": true, // Included in "Strict" 23 | // "noImplicitThis": true, // Included in "Strict" 24 | // "strictBindCallApply": true, // Included in "Strict" 25 | // "strictFunctionTypes": true, // Included in "Strict" 26 | // "strictNullChecks": true, // Included in "Strict" 27 | // "strictPropertyInitialization": true, // Included in "Strict" 28 | "noFallthroughCasesInSwitch": true, 29 | "noImplicitOverride": true, 30 | "noImplicitReturns": true, 31 | // "noUnusedLocals": true, 32 | // "noUnusedParameters": true, 33 | "useUnknownInCatchVariables": true, 34 | "noUncheckedIndexedAccess": true, // TLDR - Checking an indexed value (array[0]) now forces type as there is no confirmation that index exists 35 | // THE BELOW ARE EXTRA STRICT OPTIONS THAT SHOULD ONLY BY CONSIDERED IN VERY SAFE PROJECTS 36 | // "exactOptionalPropertyTypes": true, // TLDR - Setting to undefined is not the same as a property not being defined at all 37 | // "noPropertyAccessFromIndexSignature": true, // TLDR - Use dot notation for objects if youre sure it exists, use ['index'] notaion if unsure 38 | 39 | /* OTHER OPTIONS */ 40 | "allowSyntheticDefaultImports": true, 41 | "esModuleInterop": true, 42 | // "emitDecoratorMetadata": true, 43 | // "experimentalDecorators": true, 44 | "forceConsistentCasingInFileNames": true, 45 | "skipLibCheck": true, 46 | "useDefineForClassFields": true, 47 | 48 | "paths": { 49 | "@/*": ["./src/*"] 50 | } 51 | }, 52 | "include": ["src", "tsup.config.ts", "../reset.d.ts", "prettier.config.mjs"] 53 | } 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Upstash Search Migrator CLI 2 | 3 | A command-line tool to migrate your data from an Algolia or Meilisearch index to an [Upstash Search](https://upstash.com/docs/search) index. 4 | 5 | ## Getting Started 6 | 7 | You can run the CLI directly using `npx` without any installation, which ensures you are always using the latest version. 8 | 9 | ### Using npx 10 | 11 | ```sh 12 | npx @upstash/search-migrator 13 | ``` 14 | 15 | ## Usage 16 | 17 | The CLI can be run by passing command-line flags; otherwise, those credentials will be asked in the CLI. 18 | 19 | ### Interactive Mode 20 | 21 | Simply run the command without any flags to be guided through the migration process with interactive prompts. 22 | 23 | ```sh 24 | npx @upstash/search-migrator 25 | ``` 26 | 27 | ### Using Flags 28 | 29 | You can also provide your credentials and other information as command-line flags. 30 | 31 | #### Algolia to Upstash 32 | ```sh 33 | npx @upstash/search-migrator \ 34 | --upstash-url "YOUR_UPSTASH_URL" \ 35 | --upstash-token "YOUR_UPSTASH_TOKEN" \ 36 | --algolia-app-id "YOUR_ALGOLIA_APP_ID" \ 37 | --algolia-api-key "YOUR_ALGOLIA_WRITE_API_KEY" 38 | ``` 39 | 40 | #### Meilisearch to Upstash 41 | ```sh 42 | npx @upstash/search-migrator \ 43 | --upstash-url "YOUR_UPSTASH_URL" \ 44 | --upstash-token "YOUR_UPSTASH_TOKEN" \ 45 | --meilisearch-host "YOUR_MEILISEARCH_HOST" \ 46 | --meilisearch-api-key "YOUR_MEILISEARCH_API_KEY" 47 | ``` 48 | 49 | ## Obtaining Credentials 50 | 51 | ### Upstash 52 | 53 | 1. Go to your [Upstash Console](https://console.upstash.com/). 54 | 2. Select your Search index. 55 | 3. Under the **Details** section, you will find your `UPSTASH_SEARCH_REST_URL` and `UPSTASH_SEARCH_REST_TOKEN`. 56 | * `--upstash-url` corresponds to `UPSTASH_SEARCH_REST_URL`. 57 | * `--upstash-token` corresponds to `UPSTASH_SEARCH_REST_TOKEN`. 58 | 59 | ### Algolia 60 | 61 | 1. Go to your [Algolia Dashboard](https://www.algolia.com/dashboard). 62 | 2. Navigate to **Settings** > **API Keys**. 63 | 3. You will find your **Application ID** here. This is your `--algolia-app-id`. 64 | 4. For the API key (`--algolia-api-key`), you need a key with `write` permissions for your indices. You can use your **Write API Key** or create a new one with the necessary permissions. 65 | 66 | ### Meilisearch 67 | 68 | 1. Go to your [Meilisearch Console](https://cloud.meilisearch.com/). 69 | 2. Find your Meilisearch deployment and copy the **Host URL** and **API Key**. 70 | * `--meili-host` corresponds to your Meilisearch instance URL (e.g., `https://ms-xxxxxx.meilisearch.io`). 71 | * `--meili-api-key` corresponds to your Meilisearch API key. 72 | -------------------------------------------------------------------------------- /src/scripts/meilisearch-data-transfer.ts: -------------------------------------------------------------------------------- 1 | import { Meilisearch } from "meilisearch"; 2 | import { Search } from "@upstash/search"; 3 | import { UpstashRecord } from "../types.js"; 4 | import { mapMeilisearchToUpstash } from "../utils/mapper.js"; 5 | import { logger } from "../utils/logger.js"; 6 | 7 | 8 | async function fetchMeilisearchData(meilisearchClient: Meilisearch, indexName: string, limit: number, offset: number ) { 9 | const result = await meilisearchClient.index(indexName).getDocuments({ limit, offset }); 10 | return result; 11 | } 12 | 13 | 14 | async function upsertToUpstash(batch: UpstashRecord[], upstashIndex: any) { 15 | await upstashIndex.upsert(batch); 16 | } 17 | 18 | async function fetchAndUpsertAllData({ meilisearchClient, upstashClient, upstashIndexName, meilisearchIndexName, contentKeys }: { meilisearchClient: Meilisearch, upstashClient: Search, upstashIndexName: string, meilisearchIndexName: string, contentKeys: string[] }) { 19 | const upstashIndex = upstashClient.index(upstashIndexName); 20 | let hasMore = true; 21 | const limit = 100; 22 | let offset = 0; 23 | let totalUpserted = 0; 24 | 25 | while (hasMore) { 26 | const result = await fetchMeilisearchData(meilisearchClient, meilisearchIndexName, limit, offset); 27 | 28 | if (result.results && result.results.length > 0) { 29 | const upstashBatch = result.results.map((hit) => mapMeilisearchToUpstash(hit, contentKeys)); 30 | // Upsert in batches 31 | for (let i = 0; i < upstashBatch.length; i += limit) { 32 | const batch = upstashBatch.slice(i, i + limit); 33 | await upsertToUpstash(batch, upstashIndex); 34 | totalUpserted += batch.length; 35 | logger.info(`Upserted ${batch.length} records to Upstash (total: ${totalUpserted})`); 36 | } 37 | } 38 | if (result.total > offset + limit) { 39 | offset += limit; 40 | } else { 41 | hasMore = false; 42 | } 43 | } 44 | logger.success(`All records from Meilisearch have been upserted to Upstash. Total: ${totalUpserted}`); 45 | logger.success(`Visit https://console.upstash.com/search/ to query your data.`) 46 | } 47 | 48 | export async function main({ 49 | meilisearchClient, 50 | upstashClient, 51 | upstashIndexName, 52 | meilisearchIndexName, 53 | contentKeys, 54 | }: { 55 | meilisearchClient: Meilisearch; 56 | upstashClient: Search; 57 | upstashIndexName: string; 58 | meilisearchIndexName: string; 59 | contentKeys: string[]; 60 | }) { 61 | await fetchAndUpsertAllData({ meilisearchClient, upstashClient, upstashIndexName, meilisearchIndexName, contentKeys }); 62 | } 63 | 64 | 65 | -------------------------------------------------------------------------------- /src/scripts/algolia-data-tranfer.ts: -------------------------------------------------------------------------------- 1 | import { SearchClient } from "@algolia/client-search"; 2 | import { Search } from "@upstash/search"; 3 | import { UpstashRecord } from "../types.js"; 4 | import { mapAlgoliaToUpstash } from "../utils/mapper.js"; 5 | import { logger } from "../utils/logger.js"; 6 | 7 | 8 | async function fetchAlgoliaData(algoliaClient: SearchClient, indexName: string, cursor?: string) { 9 | const result = await algoliaClient.browse({ 10 | indexName: indexName, 11 | ...(cursor ? { browseParams: { cursor } } : {}) 12 | }); 13 | return result; 14 | } 15 | 16 | 17 | async function upsertToUpstash(batch: UpstashRecord[], upstashIndex: any) { 18 | await upstashIndex.upsert(batch); 19 | } 20 | 21 | async function fetchAndUpsertAllData({ algoliaClient, upstashClient, upstashIndexName, algoliaIndexName, contentKeys }: { algoliaClient: SearchClient, upstashClient: Search, upstashIndexName: string, algoliaIndexName: string, contentKeys: string[] }) { 22 | const upstashIndex = upstashClient.index(upstashIndexName); 23 | let cursor = undefined; 24 | let hasMore = true; 25 | const batchSize = 100; // Upstash recommends reasonable batch sizes 26 | let totalUpserted = 0; 27 | 28 | while (hasMore) { 29 | const result = await fetchAlgoliaData(algoliaClient, algoliaIndexName, cursor); 30 | 31 | if (result.hits && result.hits.length > 0) { 32 | const upstashBatch = result.hits.map((hit) => mapAlgoliaToUpstash(hit, contentKeys)); 33 | // Upsert in batches 34 | for (let i = 0; i < upstashBatch.length; i += batchSize) { 35 | const batch = upstashBatch.slice(i, i + batchSize); 36 | await upsertToUpstash(batch, upstashIndex); 37 | totalUpserted += batch.length; 38 | logger.info(`Upserted ${batch.length} records to Upstash (total: ${totalUpserted})`); 39 | } 40 | } 41 | if (result.cursor) { 42 | cursor = result.cursor; 43 | } else { 44 | hasMore = false; 45 | } 46 | } 47 | logger.success(`All records from Algolia have been upserted to Upstash. Total: ${totalUpserted}`); 48 | logger.success(`Visit https://console.upstash.com/search/ to query your data.`) 49 | } 50 | 51 | export async function main({ 52 | algoliaClient, 53 | upstashClient, 54 | upstashIndexName, 55 | algoliaIndexName, 56 | contentKeys, 57 | }: { 58 | algoliaClient: SearchClient; 59 | upstashClient: Search; 60 | upstashIndexName: string; 61 | algoliaIndexName: string; 62 | contentKeys: string[]; 63 | }) { 64 | await fetchAndUpsertAllData({ algoliaClient, upstashClient, upstashIndexName, algoliaIndexName, contentKeys }); 65 | } 66 | 67 | 68 | -------------------------------------------------------------------------------- /src/cli/migrate-from-algolia.ts: -------------------------------------------------------------------------------- 1 | import { OptionValues } from "commander" 2 | import { isCancel, outro, select, text, multiselect, spinner, confirm } from "@clack/prompts" 3 | import { createAlgoliaClient, createUpstashClient } from "../utils/client.js" 4 | import { getAlgoliaIndexFields } from "@/schemas/algolia.js" 5 | import { SearchClient } from "@algolia/client-search" 6 | import { Search } from "@upstash/search" 7 | 8 | export interface AlgoliaCliResults { 9 | algoliaClient: SearchClient 10 | upstashClient: Search 11 | upstashIndexName: string 12 | algoliaIndexName: string 13 | contentKeys: string[] 14 | } 15 | 16 | export async function runAlgoliaCli(options? : OptionValues): Promise { 17 | 18 | const upstashUrl = 19 | options?.upstashUrl ?? 20 | (await text({ 21 | message: "What is your Upstash URL?", 22 | placeholder: "https://***-gcp-***-search.upstash.io", 23 | validate: (value) => { 24 | if (!value) return "Please enter your Upstash URL" 25 | return 26 | }, 27 | })) 28 | 29 | if (isCancel(upstashUrl)) { 30 | outro("Migration cancelled.") 31 | return undefined 32 | } 33 | 34 | const upstashToken = 35 | options?.upstashToken ?? 36 | (await text({ 37 | message: "What is your Upstash Token?", 38 | placeholder: "upstash-search-token", 39 | validate: (value) => { 40 | if (!value) return "Please enter your Upstash Token" 41 | return 42 | }, 43 | })) 44 | 45 | if (isCancel(upstashToken)) { 46 | outro("Migration cancelled.") 47 | return undefined 48 | } 49 | 50 | const s = spinner() 51 | s.start("Fetching Upstash Indexes") 52 | 53 | const upstashClient = createUpstashClient(upstashUrl, upstashToken) 54 | const upstashIndexNames = await upstashClient.listIndexes() 55 | 56 | if (upstashIndexNames.length === 0) { 57 | s.stop() 58 | outro("No Upstash Indexes found. Would you like to create a new index?") 59 | const createIndex = await confirm({ 60 | message: "Create a new index?", 61 | initialValue: true, 62 | }) 63 | if (createIndex) { 64 | const indexName = await text({ 65 | message: "Enter a name for your new index", 66 | placeholder: "my-index", 67 | }) 68 | if (isCancel(indexName)) { 69 | outro("Migration cancelled.") 70 | return undefined 71 | } 72 | s.start("Creating new Upstash Index") 73 | await upstashClient.index(indexName) 74 | upstashIndexNames.push(indexName) 75 | s.stop("New Upstash Index created") 76 | } 77 | else { 78 | outro("Migration cancelled.") 79 | return undefined 80 | } 81 | } else { 82 | s.stop("Upstash Indexes fetched") 83 | } 84 | 85 | const upstashIndexName = 86 | 87 | (await select({ 88 | message: "Choose an Upstash Index to migrate to:", 89 | options: upstashIndexNames.map((name) => ({ 90 | value: name, 91 | label: name, 92 | })), 93 | })) 94 | 95 | if (isCancel(upstashIndexName)) { 96 | outro("Migration cancelled.") 97 | return undefined 98 | } 99 | 100 | const algoliaAppId = 101 | options?.algoliaAppId ?? 102 | (await text({ 103 | message: "What is your Algolia App ID?", 104 | placeholder: "algolia-app-id", 105 | validate: (value) => { 106 | if (!value) return "Please enter your Algolia App ID" 107 | return 108 | }, 109 | })) 110 | 111 | if (isCancel(algoliaAppId)) { 112 | outro("Migration cancelled.") 113 | return undefined 114 | } 115 | 116 | const algoliaApiKey = 117 | options?.algoliaApiKey ?? 118 | (await text({ 119 | message: "What is your Algolia (write) API Key?", 120 | placeholder: "algolia-api-key", 121 | validate: (value) => { 122 | if (!value) return "Please enter your Algolia API Key" 123 | return 124 | }, 125 | })) 126 | 127 | if (isCancel(algoliaApiKey)) { 128 | outro("Migration cancelled.") 129 | return undefined 130 | } 131 | 132 | s.start("Fetching Algolia Indexes") 133 | 134 | const algoliaClient = createAlgoliaClient(algoliaAppId, algoliaApiKey) 135 | const algoliaIndices = await algoliaClient.listIndices() 136 | const algoliaIndexNames = algoliaIndices.items.map((index) => index.name) 137 | 138 | if (algoliaIndexNames.length === 0) { 139 | s.stop() 140 | outro("No Algolia Indexes found. Please create an index first.") 141 | return undefined 142 | } 143 | 144 | s.stop("Algolia Indexes fetched") 145 | 146 | const algoliaIndexName = 147 | 148 | (await select({ 149 | message: "Choose an Algolia Index to migrate from:", 150 | options: algoliaIndexNames.map((name) => ({ 151 | value: name, 152 | label: name, 153 | })), 154 | })) 155 | 156 | if (isCancel(algoliaIndexName)) { 157 | outro("Migration cancelled.") 158 | return undefined 159 | } 160 | 161 | const indexFields = await getAlgoliaIndexFields(algoliaClient, algoliaIndexName) 162 | 163 | const contentKeys = await multiselect({ 164 | message: "Select fields to include in content (use space bar to select)", 165 | options: indexFields.map((key) => ({ value: key, label: key })), 166 | required: true 167 | }) as string[] 168 | 169 | if (isCancel(contentKeys)) { 170 | outro("Migration cancelled.") 171 | return undefined 172 | } 173 | 174 | const contentKeysArray = contentKeys.map((key) => key.trim()) 175 | 176 | 177 | return { 178 | algoliaClient, 179 | upstashClient, 180 | upstashIndexName, 181 | algoliaIndexName, 182 | contentKeys: contentKeysArray, 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/cli/migrate-from-meilisearch.ts: -------------------------------------------------------------------------------- 1 | import { OptionValues } from "commander" 2 | import { isCancel, outro, select, text, multiselect, spinner, confirm } from "@clack/prompts" 3 | import { createMeilisearchClient, createUpstashClient } from "../utils/client.js" 4 | import { getMeilisearchIndexFields } from "../schemas/meilisearch.js" 5 | import { Meilisearch } from "meilisearch"; 6 | import { Search } from "@upstash/search" 7 | 8 | export interface MeilisearchCliResults { 9 | meilisearchClient: Meilisearch 10 | upstashClient: Search 11 | upstashIndexName: string 12 | meilisearchIndexName: string 13 | contentKeys: string[] 14 | } 15 | 16 | export async function runMeilisearchCli(options? : OptionValues): Promise { 17 | 18 | const upstashUrl = 19 | options?.upstashUrl ?? 20 | (await text({ 21 | message: "What is your Upstash URL?", 22 | placeholder: "https://***-gcp-***-search.upstash.io", 23 | validate: (value) => { 24 | if (!value) return "Please enter your Upstash URL" 25 | return 26 | }, 27 | })) 28 | 29 | if (isCancel(upstashUrl)) { 30 | outro("Migration cancelled.") 31 | return undefined 32 | } 33 | 34 | const upstashToken = 35 | options?.upstashToken ?? 36 | (await text({ 37 | message: "What is your Upstash Token?", 38 | placeholder: "upstash-search-token", 39 | validate: (value) => { 40 | if (!value) return "Please enter your Upstash Token" 41 | return 42 | }, 43 | })) 44 | 45 | if (isCancel(upstashToken)) { 46 | outro("Migration cancelled.") 47 | return undefined 48 | } 49 | 50 | const s = spinner() 51 | s.start("Fetching Upstash Indexes") 52 | 53 | const upstashClient = createUpstashClient(upstashUrl, upstashToken) 54 | const upstashIndexNames = await upstashClient.listIndexes() 55 | 56 | if (upstashIndexNames.length === 0) { 57 | s.stop() 58 | outro("No Upstash Indexes found. Would you like to create a new index?") 59 | const createIndex = await confirm({ 60 | message: "Create a new index?", 61 | initialValue: true, 62 | }) 63 | if (createIndex) { 64 | const indexName = await text({ 65 | message: "Enter a name for your new index", 66 | placeholder: "my-index", 67 | }) 68 | if (isCancel(indexName)) { 69 | outro("Migration cancelled.") 70 | return undefined 71 | } 72 | s.start("Creating new Upstash Index") 73 | await upstashClient.index(indexName) 74 | upstashIndexNames.push(indexName) 75 | s.stop("New Upstash Index created") 76 | } 77 | else { 78 | outro("Migration cancelled.") 79 | return undefined 80 | } 81 | } else { 82 | s.stop("Upstash Indexes fetched") 83 | } 84 | 85 | const upstashIndexName = 86 | 87 | (await select({ 88 | message: "Choose an Upstash Index to migrate to:", 89 | options: upstashIndexNames.map((name) => ({ 90 | value: name, 91 | label: name, 92 | })), 93 | })) 94 | 95 | if (isCancel(upstashIndexName)) { 96 | outro("Migration cancelled.") 97 | return undefined 98 | } 99 | 100 | const meilisearchHost = 101 | options?.meilisearchHost ?? 102 | (await text({ 103 | message: "What is your Meilisearch Host?", 104 | placeholder: "https://***.meilisearch.io", 105 | validate: (value) => { 106 | if (!value) return "Please enter your Meilisearch Host" 107 | return 108 | }, 109 | })) 110 | 111 | if (isCancel(meilisearchHost)) { 112 | outro("Migration cancelled.") 113 | return undefined 114 | } 115 | 116 | const meilisearchApiKey = 117 | options?.meilisearchApiKey ?? 118 | (await text({ 119 | message: "What is your Meilisearch API Key?", 120 | placeholder: "meilisearch-api-key", 121 | validate: (value) => { 122 | if (!value) return "Please enter your Meilisearch API Key" 123 | return 124 | }, 125 | })) 126 | 127 | if (isCancel(meilisearchApiKey)) { 128 | outro("Migration cancelled.") 129 | return undefined 130 | } 131 | 132 | s.start("Fetching Meilisearch Indexes") 133 | 134 | const meilisearchClient = createMeilisearchClient(meilisearchHost, meilisearchApiKey) 135 | const meilisearchIndexResults = (await meilisearchClient.getRawIndexes()).results 136 | const meilisearchIndexNames = meilisearchIndexResults.map((index) => index.uid) 137 | 138 | if (meilisearchIndexNames.length === 0) { 139 | s.stop() 140 | outro("No Meilisearch Indexes found. Please create an index first.") 141 | return undefined 142 | } 143 | 144 | s.stop("Meilisearch Indexes fetched") 145 | 146 | const meilisearchIndexName = 147 | (await select({ 148 | message: "Choose a Meilisearch Index to migrate from:", 149 | options: meilisearchIndexNames.map((name) => ({ 150 | value: name, 151 | label: name, 152 | })), 153 | })) 154 | 155 | if (isCancel(meilisearchIndexName)) { 156 | outro("Migration cancelled.") 157 | return undefined 158 | } 159 | 160 | const indexFields = await getMeilisearchIndexFields(meilisearchClient, meilisearchIndexName) 161 | 162 | const contentKeys = await multiselect({ 163 | message: "Select fields to include in content (use space bar to select)", 164 | options: indexFields.map((key) => ({ value: key, label: key })), 165 | required: true 166 | }) as string[] 167 | 168 | if (isCancel(contentKeys)) { 169 | outro("Migration cancelled.") 170 | return undefined 171 | } 172 | 173 | const contentKeysArray = contentKeys.map((key) => key.trim()) 174 | 175 | 176 | return { 177 | meilisearchClient, 178 | upstashClient, 179 | upstashIndexName, 180 | meilisearchIndexName, 181 | contentKeys: contentKeysArray, 182 | } 183 | } 184 | --------------------------------------------------------------------------------