├── app ├── globals.css ├── layout.tsx └── page.tsx ├── postcss.config.mjs ├── next.config.ts ├── .gitignore ├── tsconfig.json ├── package.json ├── pages └── api │ └── crawl.ts ├── README.md └── components ├── SearchComponent.tsx └── RecentUpdates.tsx /app/globals.css: -------------------------------------------------------------------------------- 1 | @import "tailwindcss"; 2 | -------------------------------------------------------------------------------- /postcss.config.mjs: -------------------------------------------------------------------------------- 1 | const config = { 2 | plugins: ["@tailwindcss/postcss"], 3 | }; 4 | 5 | export default config; 6 | -------------------------------------------------------------------------------- /next.config.ts: -------------------------------------------------------------------------------- 1 | import type { NextConfig } from "next"; 2 | 3 | const nextConfig: NextConfig = { 4 | /* config options here */ 5 | }; 6 | 7 | export default nextConfig; 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.* 7 | .yarn/* 8 | !.yarn/patches 9 | !.yarn/plugins 10 | !.yarn/releases 11 | !.yarn/versions 12 | 13 | # testing 14 | /coverage 15 | 16 | # next.js 17 | /.next/ 18 | /out/ 19 | 20 | # production 21 | /build 22 | 23 | # misc 24 | .DS_Store 25 | *.pem 26 | 27 | # debug 28 | npm-debug.log* 29 | yarn-debug.log* 30 | yarn-error.log* 31 | .pnpm-debug.log* 32 | 33 | # env files (can opt-in for committing if needed) 34 | .env* 35 | 36 | # vercel 37 | .vercel 38 | 39 | # typescript 40 | *.tsbuildinfo 41 | next-env.d.ts 42 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "noEmit": true, 9 | "esModuleInterop": true, 10 | "module": "esnext", 11 | "moduleResolution": "bundler", 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "jsx": "preserve", 15 | "incremental": true, 16 | "plugins": [ 17 | { 18 | "name": "next" 19 | } 20 | ], 21 | "paths": { 22 | "@/*": ["./*"] 23 | } 24 | }, 25 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 26 | "exclude": ["node_modules"] 27 | } 28 | -------------------------------------------------------------------------------- /app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from "next"; 2 | import { Geist, Geist_Mono } from "next/font/google"; 3 | import "./globals.css"; 4 | 5 | const geistSans = Geist({ 6 | variable: "--font-geist-sans", 7 | subsets: ["latin"], 8 | }); 9 | 10 | const geistMono = Geist_Mono({ 11 | variable: "--font-geist-mono", 12 | subsets: ["latin"], 13 | }); 14 | 15 | export const metadata: Metadata = { 16 | title: "Upstash Docs Library", 17 | description: "A modern documentation library to search and track the docs you like.", 18 | }; 19 | 20 | export default function RootLayout({ 21 | children, 22 | }: Readonly<{ 23 | children: React.ReactNode; 24 | }>) { 25 | return ( 26 | 27 | 30 | {children} 31 | 32 | 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "documentation-library", 3 | "version": "0.1.0", 4 | "description": "A modern documentation library to search and track the docs.", 5 | "private": true, 6 | "scripts": { 7 | "dev": "next dev --turbopack", 8 | "build": "next build", 9 | "start": "next start", 10 | "lint": "next lint" 11 | }, 12 | "dependencies": { 13 | "@upstash/search": "^0.1.3", 14 | "@upstash/search-crawler": "^0.1.8", 15 | "@upstash/search-ui": "^0.1.4", 16 | "lucide-react": "^0.525.0", 17 | "next": "15.3.8", 18 | "react": "^19.0.0", 19 | "react-dom": "^19.0.0" 20 | }, 21 | "devDependencies": { 22 | "@tailwindcss/postcss": "^4", 23 | "@types/node": "^20", 24 | "@types/react": "^19", 25 | "@types/react-dom": "^19", 26 | "tailwindcss": "^4", 27 | "tw-animate-css": "^1.3.5", 28 | "typescript": "^5" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pages/api/crawl.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next' 2 | import { crawlAndIndex } from "@upstash/search-crawler" 3 | 4 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 5 | if (req.method === 'POST') { 6 | const { docsUrl, index } = req.body; 7 | const upstashUrl = process.env.NEXT_PUBLIC_UPSTASH_SEARCH_URL! 8 | const upstashRestToken = process.env.UPSTASH_SEARCH_REST_TOKEN! 9 | 10 | if (!docsUrl || !upstashUrl || !upstashRestToken) { 11 | return res.status(500).json({ error: "Missing Upstash Search configuration" }); 12 | } 13 | 14 | const result = await crawlAndIndex({ 15 | upstashUrl, 16 | upstashToken: upstashRestToken, 17 | indexName: index || "default", 18 | docUrl: docsUrl, 19 | }) 20 | 21 | return res.status(200).json(result) 22 | } else { 23 | return res.status(405).json({ error: "Method not allowed" }); 24 | } 25 | } -------------------------------------------------------------------------------- /app/page.tsx: -------------------------------------------------------------------------------- 1 | import SearchComponent from "../components/SearchComponent" 2 | import RecentUpdates from "../components/RecentUpdates" 3 | import { BookOpen } from "lucide-react" 4 | 5 | export default function Page() { 6 | return ( 7 |
8 |
9 |
10 |
11 |
12 | 13 |

Documentation Library

14 |
15 |

16 | Search across all your documentation sources and discover the latest updates 17 |

18 |
19 |
20 |
21 | 22 |
23 | 24 | 25 |
26 |
27 | ) 28 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fupstash%2Fsearch-docs&env=NEXT_PUBLIC_UPSTASH_SEARCH_URL,NEXT_PUBLIC_UPSTASH_SEARCH_READONLY_TOKEN,UPSTASH_SEARCH_REST_TOKEN&envDescription=Credentials%20needed%20for%20Upstash%20Search%20Component%20use&envLink=https%3A%2F%2Fconsole.upstash.com%2Fsearch&project-name=search-docs&repository-name=search-docs&demo-title=Documentation%20Library&demo-description=Search%20across%20all%20your%20documentation%20sources%20and%20discover%20the%20latest%20updates&demo-url=https%3A%2F%2Fsearch-docs.vercel.app%2F) 2 | ## Description 3 | 4 | A modern documentation library to search and track the docs. 5 | 6 | ## Getting Started 7 | 8 | First, run the development server: 9 | 10 | ```bash 11 | npm install 12 | npm run dev 13 | ``` 14 | 15 | ## How it Works 16 | - Search: Uses Upstash Search UI to query multiple indexes in parallel, sorts and groups results, and displays them with section headers. 17 | - Recent Updates: Upstash Qstash fetches all documents from multiple indexes in batches, filters for those crawled in the last 24 hours. 18 | 19 | ## Set the Crawler 20 | 21 | - Upstash Qstash can call this endpoint: `/api/crawl` on schedule to crawl the relevant data 22 | - Providing the URL and the index name in the body, you may manage the crawler, 23 | e.g. 24 | 25 | ``` 26 | { 27 | "docsUrl": "https://nextjs.org/docs", 28 | "index": "next-js" 29 | } 30 | ``` 31 | 32 | ## Conclusion 33 | Finally, the UI will make use of these components to serve users to find whatever they want from 34 | any source they want. Moreover, they can keep up with the updates in their favorite docs. 35 | 36 | 37 | -------------------------------------------------------------------------------- /components/SearchComponent.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import { SearchBar } from "@upstash/search-ui" 4 | import "@upstash/search-ui/dist/index.css" 5 | import { Search } from "@upstash/search" 6 | import { FileText } from "lucide-react" 7 | 8 | // Initialize Upstash Search client 9 | const client = new Search({ 10 | url: process.env.NEXT_PUBLIC_UPSTASH_SEARCH_URL || "", 11 | token: process.env.NEXT_PUBLIC_UPSTASH_SEARCH_READONLY_TOKEN || "", 12 | }) 13 | 14 | interface SearchResult { 15 | id: string 16 | content: { 17 | title: string 18 | fullContent: string 19 | } 20 | metadata: { 21 | url: string 22 | path: string 23 | contentLength: number 24 | crawledAt: string 25 | } 26 | score: number 27 | indexName?: string 28 | } 29 | 30 | 31 | async function searchDocs(query: string): Promise { 32 | if (!query.trim()) return [] 33 | 34 | try { 35 | const indexes = await client.listIndexes() 36 | 37 | const searchPromises = indexes.map(async (indexName) => { 38 | try { 39 | const index = client.index(indexName) 40 | const searchParams: any = { 41 | query, 42 | limit: 10, 43 | reranking: true 44 | } 45 | 46 | const results = await index.search(searchParams) 47 | 48 | return (results as any[]).map((result, i) => ({ 49 | ...result, 50 | id: `${indexName}-${result.id}`, 51 | indexName 52 | })) 53 | } catch (error) { 54 | console.error(`Error searching ${indexName}:`, error) 55 | return [] 56 | } 57 | }) 58 | 59 | const resultArrays = await Promise.all(searchPromises) 60 | 61 | const allResults = resultArrays.flat() as SearchResult[] 62 | 63 | const topResults = allResults 64 | .sort((a, b) => (b.score || 0) - (a.score || 0)) 65 | .slice(0, 10) 66 | 67 | return topResults 68 | 69 | } catch (error) { 70 | console.error('Search error:', error) 71 | return [] 72 | } 73 | } 74 | 75 | 76 | export default function SearchComponent() { 77 | 78 | return ( 79 |
80 | {/* Search Bar */} 81 |
82 | 83 | 84 | 85 | 86 | 87 | 90 | {(result) => ( 91 | 92 | 93 | 94 | 95 | 96 | 97 | { 98 | window.open(result.metadata?.url, "_blank") 99 | }}> 100 | {result.content?.title} 101 | 102 |

{result.indexName}

103 | 104 | 105 | )} 106 | 107 | 108 | 109 |
110 |
111 | ) 112 | } -------------------------------------------------------------------------------- /components/RecentUpdates.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import { Search } from "@upstash/search" 4 | import { FileText, Clock } from "lucide-react" 5 | import { useState, useEffect } from "react" 6 | 7 | // Initialize Upstash Search client 8 | const client = new Search({ 9 | url: process.env.NEXT_PUBLIC_UPSTASH_SEARCH_URL || "", 10 | token: process.env.NEXT_PUBLIC_UPSTASH_SEARCH_READONLY_TOKEN || "", 11 | }) 12 | 13 | interface SearchResult { 14 | id: string 15 | content: { 16 | title: string 17 | fullContent: string 18 | } 19 | metadata: { 20 | url: string 21 | path: string 22 | contentLength: number 23 | crawledAt: string 24 | } 25 | score: number 26 | indexName?: string 27 | } 28 | 29 | async function getLatestDocs(): Promise { 30 | try { 31 | const indexes = await client.listIndexes() 32 | 33 | const currentDate = new Date() 34 | currentDate.setDate(currentDate.getDate() - 7) 35 | const oneDayAgo = currentDate.getTime() 36 | 37 | const rangePromises = indexes.map(async (indexName) => { 38 | try { 39 | const index = client.index(indexName) 40 | let recentDocuments: SearchResult[] = [] 41 | let cursor = "" 42 | 43 | while (true) { 44 | const results = await index.range({ 45 | cursor: cursor, 46 | limit: 100 47 | }) 48 | //TODO: use metadata filter instead as param 49 | const recentBatch = results.documents 50 | .filter(result => { 51 | if (!result.metadata?.crawledAt) return false 52 | const crawledTime = new Date(result.metadata.crawledAt as string).getTime() 53 | return crawledTime >= oneDayAgo 54 | }) 55 | .map((result) => ({ 56 | ...result, 57 | id: `${indexName}-${result.id}`, 58 | indexName 59 | })) as SearchResult[] 60 | 61 | recentDocuments = recentDocuments.concat(recentBatch) 62 | 63 | if (!results.nextCursor || results.nextCursor === cursor || recentDocuments.length >= 10) { 64 | break 65 | } 66 | cursor = results.nextCursor 67 | } 68 | 69 | return recentDocuments.slice(0, 10) 70 | } catch (error) { 71 | console.error(`Error getting documents from ${indexName}:`, error) 72 | return [] 73 | } 74 | }) 75 | 76 | const allResultArrays = await Promise.all(rangePromises) 77 | 78 | const allResults = allResultArrays.flat() as SearchResult[] 79 | 80 | return allResults 81 | 82 | } catch (error) { 83 | console.error('Error getting latest docs:', error) 84 | return [] 85 | } 86 | } 87 | 88 | export default function RecentUpdates() { 89 | const [latestDocs, setLatestDocs] = useState([]) 90 | const [loadingLatest, setLoadingLatest] = useState(true) 91 | 92 | useEffect(() => { 93 | const loadLatestDocs = async () => { 94 | const latest = await getLatestDocs() 95 | setLatestDocs(latest) 96 | setLoadingLatest(false) 97 | } 98 | 99 | loadingLatest && loadLatestDocs() 100 | }, [latestDocs]) 101 | 102 | const formatDate = (dateString: string) => { 103 | const date = new Date(dateString) 104 | return date.toLocaleDateString() + " " + date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) 105 | } 106 | 107 | return ( 108 |
109 |
110 |

111 | 112 | Recent Updates 113 |

114 |
115 | 116 |
117 | {loadingLatest ? ( 118 |
119 |
120 | Loading... 121 |
122 | ) : ( 123 |
124 |
125 | {latestDocs.map((doc, index) => ( 126 |
127 |
window.open(doc.metadata?.url, "_blank")} 130 | > 131 |
132 | 133 |
134 |
135 |

136 | {doc.content?.title || 'Documentation'} 137 |

138 |
139 | 140 | {doc.indexName} 141 | 142 | 143 | {formatDate(doc.metadata.crawledAt)} 144 | 145 |
146 |
147 |
148 | {index < latestDocs.length - 1 && ( 149 |
150 | )} 151 |
152 | ))} 153 | 154 | {latestDocs.length === 0 && ( 155 |
156 | 157 |

No recent updates in the last week

158 |
159 | )} 160 |
161 |
162 | )} 163 |
164 |
165 | ) 166 | } --------------------------------------------------------------------------------