Vector Database Implementation Guide

Complete step-by-step guide to implementing semantic code search with Claude Code and vector databases.

🎯 Project Setup

Prerequisites

# Required tools
node --version  # 18+
npm --version   # 9+
git --version   # 2.30+
 
# Optional (for self-hosted)
docker --version  # 20+

Initialize Project

# Create project
mkdir claude-code-search
cd claude-code-search
npm init -y
 
# Install dependencies
npm install @anthropic-ai/claude-code-sdk
npm install @pinecone-database/pinecone
npm install @babel/parser @babel/traverse
npm install langchain openai
npm install dotenv commander
 
# Dev dependencies
npm install -D typescript @types/node
npm install -D vitest @vitest/ui
npm install -D eslint prettier

🏗️ Project Structure

claude-code-search/
├── src/
│   ├── index.ts           # Main entry point
│   ├── embeddings/        # Embedding generation
│   │   ├── index.ts
│   │   ├── code-embedder.ts
│   │   └── ast-analyzer.ts
│   ├── indexing/          # Code indexing
│   │   ├── index.ts
│   │   ├── file-crawler.ts
│   │   └── incremental.ts
│   ├── search/            # Search implementation
│   │   ├── index.ts
│   │   ├── semantic-search.ts
│   │   └── reranker.ts
│   ├── databases/         # Vector DB adapters
│   │   ├── index.ts
│   │   ├── pinecone.ts
│   │   ├── weaviate.ts
│   │   └── qdrant.ts
│   └── utils/            # Utilities
│       ├── cache.ts
│       └── logger.ts
├── config/
│   ├── default.json
│   └── production.json
├── scripts/
│   ├── index-codebase.ts
│   └── test-search.ts
└── tests/
    └── search.test.ts

📝 Core Implementation

1. Code Embedder

// src/embeddings/code-embedder.ts
import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
import { ClaudeCode } from '@anthropic-ai/claude-code-sdk';
import { parse } from '@babel/parser';
import traverse from '@babel/traverse';
 
export class CodeEmbedder {
  private embeddings: OpenAIEmbeddings;
  private claude: ClaudeCode;
  
  constructor() {
    this.embeddings = new OpenAIEmbeddings({
      openAIApiKey: process.env.OPENAI_API_KEY,
      modelName: 'text-embedding-3-large'
    });
    
    this.claude = new ClaudeCode({
      apiKey: process.env.ANTHROPIC_API_KEY
    });
  }
 
  async embedCode(
    code: string, 
    metadata: CodeMetadata
  ): Promise<EmbeddingResult> {
    // Extract AST features
    const astFeatures = this.extractASTFeatures(code, metadata.language);
    
    // Get Claude's understanding
    const understanding = await this.getCodeUnderstanding(code);
    
    // Combine all context
    const enrichedText = this.combineContext(
      code,
      astFeatures,
      understanding,
      metadata
    );
    
    // Generate embedding
    const vector = await this.embeddings.embedQuery(enrichedText);
    
    return {
      vector,
      metadata: {
        ...metadata,
        astFeatures,
        understanding,
        timestamp: Date.now()
      }
    };
  }
 
  private extractASTFeatures(code: string, language: string) {
    if (language !== 'javascript' && language !== 'typescript') {
      return {};
    }
 
    try {
      const ast = parse(code, {
        sourceType: 'module',
        plugins: ['typescript', 'jsx']
      });
 
      const features = {
        imports: [] as string[],
        exports: [] as string[],
        classes: [] as string[],
        functions: [] as string[],
        complexity: 0
      };
 
      traverse(ast, {
        ImportDeclaration(path) {
          features.imports.push(path.node.source.value);
        },
        ExportNamedDeclaration(path) {
          features.exports.push('named');
        },
        ClassDeclaration(path) {
          if (path.node.id) {
            features.classes.push(path.node.id.name);
          }
        },
        FunctionDeclaration(path) {
          if (path.node.id) {
            features.functions.push(path.node.id.name);
          }
        }
      });
 
      return features;
    } catch (error) {
      console.error('AST parsing failed:', error);
      return {};
    }
  }
 
  private async getCodeUnderstanding(code: string): Promise<string> {
    const response = await this.claude.complete({
      prompt: `Analyze this code and provide a brief summary of:
        1. Main purpose and functionality
        2. Key patterns or algorithms used
        3. Notable features or techniques
        
        Code:
        ${code.slice(0, 2000)} // Limit for context
        
        Summary (2-3 sentences):`,
      max_tokens: 150
    });
 
    return response.text;
  }
 
  private combineContext(
    code: string,
    astFeatures: any,
    understanding: string,
    metadata: CodeMetadata
  ): string {
    return `
File: ${metadata.filepath}
Language: ${metadata.language}
 
Purpose: ${understanding}
 
Code Structure:
- Imports: ${astFeatures.imports?.join(', ') || 'none'}
- Exports: ${astFeatures.exports?.length || 0} exports
- Classes: ${astFeatures.classes?.join(', ') || 'none'}
- Functions: ${astFeatures.functions?.join(', ') || 'none'}
 
Code Content:
${code.slice(0, 3000)}
    `.trim();
  }
}

2. File Indexer

// src/indexing/file-crawler.ts
import { glob } from 'glob';
import fs from 'fs/promises';
import path from 'path';
import { createHash } from 'crypto';
 
export class FileCrawler {
  private patterns: string[];
  private ignorePatterns: string[];
  
  constructor(config: CrawlerConfig) {
    this.patterns = config.patterns || ['**/*.{js,ts,jsx,tsx,py,java,go}'];
    this.ignorePatterns = config.ignore || [
      '**/node_modules/**',
      '**/dist/**',
      '**/.git/**',
      '**/coverage/**'
    ];
  }
 
  async crawl(rootDir: string): AsyncGenerator<FileInfo> {
    const files = await glob(this.patterns, {
      cwd: rootDir,
      ignore: this.ignorePatterns,
      absolute: true
    });
 
    for (const filepath of files) {
      try {
        const content = await fs.readFile(filepath, 'utf-8');
        const stats = await fs.stat(filepath);
        
        yield {
          filepath,
          content,
          language: this.detectLanguage(filepath),
          size: stats.size,
          modified: stats.mtime,
          hash: this.hashContent(content)
        };
      } catch (error) {
        console.error(`Failed to read ${filepath}:`, error);
      }
    }
  }
 
  private detectLanguage(filepath: string): string {
    const ext = path.extname(filepath).toLowerCase();
    const languageMap: Record<string, string> = {
      '.js': 'javascript',
      '.jsx': 'javascript',
      '.ts': 'typescript',
      '.tsx': 'typescript',
      '.py': 'python',
      '.java': 'java',
      '.go': 'go',
      '.rs': 'rust',
      '.cpp': 'cpp',
      '.c': 'c'
    };
    
    return languageMap[ext] || 'unknown';
  }
 
  private hashContent(content: string): string {
    return createHash('sha256').update(content).digest('hex');
  }
}

3. Vector Database Adapter

// src/databases/pinecone.ts
import { PineconeClient } from '@pinecone-database/pinecone';
 
export class PineconeAdapter implements VectorDatabase {
  private client: PineconeClient;
  private indexName: string;
  
  constructor(config: PineconeConfig) {
    this.client = new PineconeClient();
    this.indexName = config.indexName;
  }
 
  async initialize() {
    await this.client.init({
      apiKey: process.env.PINECONE_API_KEY!,
      environment: process.env.PINECONE_ENVIRONMENT!
    });
 
    // Create index if it doesn't exist
    const indexes = await this.client.listIndexes();
    if (!indexes.includes(this.indexName)) {
      await this.client.createIndex({
        name: this.indexName,
        dimension: 3072, // text-embedding-3-large
        metric: 'cosine',
        podType: 'p1.x1'
      });
    }
  }
 
  async upsert(items: EmbeddingItem[]): Promise<void> {
    const index = this.client.Index(this.indexName);
    
    // Batch upserts for efficiency
    const batches = this.chunk(items, 100);
    
    for (const batch of batches) {
      const vectors = batch.map(item => ({
        id: item.id,
        values: item.vector,
        metadata: item.metadata
      }));
      
      await index.upsert({ vectors });
    }
  }
 
  async search(
    vector: number[], 
    options: SearchOptions
  ): Promise<SearchResult[]> {
    const index = this.client.Index(this.indexName);
    
    const results = await index.query({
      vector,
      topK: options.limit || 10,
      includeMetadata: true,
      filter: this.buildFilter(options.filters)
    });
 
    return results.matches.map(match => ({
      id: match.id,
      score: match.score,
      metadata: match.metadata
    }));
  }
 
  private buildFilter(filters?: SearchFilters) {
    if (!filters) return undefined;
    
    const conditions: any = {};
    
    if (filters.language) {
      conditions.language = { $eq: filters.language };
    }
    
    if (filters.minSize) {
      conditions.size = { $gte: filters.minSize };
    }
    
    if (filters.modifiedAfter) {
      conditions.modified = { 
        $gte: new Date(filters.modifiedAfter).getTime() 
      };
    }
    
    return conditions;
  }
 
  private chunk<T>(array: T[], size: number): T[][] {
    const chunks: T[][] = [];
    for (let i = 0; i < array.length; i += size) {
      chunks.push(array.slice(i, i + size));
    }
    return chunks;
  }
}

4. Semantic Search Engine

// src/search/semantic-search.ts
import { ClaudeCode } from '@anthropic-ai/claude-code-sdk';
 
export class SemanticSearchEngine {
  constructor(
    private embedder: CodeEmbedder,
    private vectorDB: VectorDatabase,
    private claude: ClaudeCode
  ) {}
 
  async search(query: string, options: SearchOptions = {}) {
    // Step 1: Enhance query
    const enhancedQuery = await this.enhanceQuery(query);
    
    // Step 2: Generate query embedding
    const queryVector = await this.embedder.embedQuery(enhancedQuery);
    
    // Step 3: Vector search
    const candidates = await this.vectorDB.search(queryVector, {
      limit: options.limit ? options.limit * 3 : 30, // Over-fetch for reranking
      filters: options.filters
    });
    
    // Step 4: Load full content
    const withContent = await this.loadContent(candidates);
    
    // Step 5: Rerank with Claude
    const reranked = await this.rerank(query, withContent);
    
    // Step 6: Format results
    return this.formatResults(reranked, options.limit || 10);
  }
 
  private async enhanceQuery(query: string): Promise<string> {
    const response = await this.claude.complete({
      prompt: `Enhance this code search query by adding related terms, 
        synonyms, and common implementations:
        
        Original query: "${query}"
        
        Enhanced query (add relevant terms, keep original):`,
      max_tokens: 100
    });
 
    return `${query} ${response.text}`;
  }
 
  private async rerank(
    query: string, 
    candidates: SearchResultWithContent[]
  ): Promise<RankedResult[]> {
    const response = await this.claude.complete({
      prompt: `Given the search query "${query}", rank these code snippets 
        by relevance (most relevant first).
        
        Consider:
        - Exact functionality match
        - Code quality and patterns
        - Modern best practices
        
        ${candidates.map((c, i) => `
        [${i}] ${c.metadata.filepath}
        Score: ${c.score}
        Preview: ${c.content.slice(0, 200)}...
        `).join('\n')}
        
        Return rankings as: [index1, index2, ...]`,
      max_tokens: 50
    });
 
    const rankings = this.parseRankings(response.text);
    
    return rankings.map((idx, rank) => ({
      ...candidates[idx],
      rank,
      relevanceScore: candidates[idx].score * (1 - rank * 0.1)
    }));
  }
 
  private async loadContent(
    results: SearchResult[]
  ): Promise<SearchResultWithContent[]> {
    return Promise.all(
      results.map(async (result) => {
        const content = await this.loadFile(result.metadata.filepath);
        return { ...result, content };
      })
    );
  }
 
  private formatResults(
    results: RankedResult[], 
    limit: number
  ): FormattedSearchResult[] {
    return results.slice(0, limit).map(result => ({
      filepath: result.metadata.filepath,
      language: result.metadata.language,
      score: result.relevanceScore,
      preview: this.generatePreview(result.content),
      explanation: result.metadata.understanding,
      highlights: this.extractHighlights(result.content)
    }));
  }
}

🚀 Usage Examples

Index a Codebase

// scripts/index-codebase.ts
import { CodeIndexer } from '../src/indexing';
import { PineconeAdapter } from '../src/databases/pinecone';
import { CodeEmbedder } from '../src/embeddings';
 
async function indexCodebase(rootDir: string) {
  // Initialize components
  const embedder = new CodeEmbedder();
  const vectorDB = new PineconeAdapter({ 
    indexName: 'code-search' 
  });
  await vectorDB.initialize();
  
  const indexer = new CodeIndexer(embedder, vectorDB);
  
  // Index with progress tracking
  await indexer.indexDirectory(rootDir, {
    onProgress: (progress) => {
      console.log(`Indexed ${progress.completed}/${progress.total} files`);
    },
    incremental: true // Only index changed files
  });
  
  console.log('Indexing complete!');
}
 
// Run indexing
indexCodebase(process.argv[2] || '.');

Search Interface

// scripts/search-cli.ts
import { Command } from 'commander';
import { SemanticSearchEngine } from '../src/search';
 
const program = new Command();
 
program
  .name('code-search')
  .description('Semantic code search powered by Claude')
  .version('1.0.0');
 
program
  .command('search <query>')
  .option('-l, --limit <number>', 'result limit', '10')
  .option('--language <lang>', 'filter by language')
  .option('--path <pattern>', 'filter by path pattern')
  .action(async (query, options) => {
    const search = new SemanticSearchEngine(
      embedder,
      vectorDB,
      claude
    );
 
    const results = await search.search(query, {
      limit: parseInt(options.limit),
      filters: {
        language: options.language,
        pathPattern: options.path
      }
    });
 
    results.forEach((result, idx) => {
      console.log(`\n[${idx + 1}] ${result.filepath}`);
      console.log(`Score: ${result.score.toFixed(3)}`);
      console.log(`Language: ${result.language}`);
      console.log(`Preview:\n${result.preview}`);
      console.log('---');
    });
  });
 
program.parse();

🧪 Testing

// tests/search.test.ts
import { describe, it, expect, beforeAll } from 'vitest';
import { SemanticSearchEngine } from '../src/search';
 
describe('Semantic Code Search', () => {
  let searchEngine: SemanticSearchEngine;
  
  beforeAll(async () => {
    searchEngine = await createTestSearchEngine();
  });
 
  it('should find authentication functions', async () => {
    const results = await searchEngine.search(
      'authentication middleware JWT validation'
    );
 
    expect(results).toHaveLength(10);
    expect(results[0].filepath).toContain('auth');
    expect(results[0].score).toBeGreaterThan(0.8);
  });
 
  it('should filter by language', async () => {
    const results = await searchEngine.search('database connection', {
      filters: { language: 'typescript' }
    });
 
    results.forEach(result => {
      expect(result.language).toBe('typescript');
    });
  });
 
  it('should understand behavioral queries', async () => {
    const results = await searchEngine.search(
      'functions that retry failed operations'
    );
 
    const topResult = results[0];
    expect(topResult.content).toMatch(/retry|exponential|backoff/i);
  });
});

🔗 Next Steps

• View Example Implementations
• Performance Optimization Guide
• Performance Patterns
• Back to Documentation

🗺️ Navigation

← Vector Databases | Examples → | Optimization →