Llama 3.2 11B Vision: Technical Analysis

const express = require('express');
const fs = require('fs');
const path = require('path');
const multer = require('multer');

class LlamaVisionServer {
    constructor() {
        this.app = express();
        this.setupMiddleware();
        this.setupRoutes();
        this.port = process.env.PORT || 3000;
    }

    setupMiddleware() {
        this.app.use(express.json({ limit: '50mb' }));
        this.app.use(express.urlencoded({ extended: true, limit: '50mb' }));

        // File upload handling for images
        const storage = multer.diskStorage({
            destination: (req, file, cb) => {
                cb(null, 'uploads/');
            },
            filename: (req, file, cb) => {
                cb(null, Date.now() + '-' + file.originalname);
            }
        });

        this.upload = multer({ storage });
    }

    setupRoutes() {
        // Health check
        this.app.get('/health', (req, res) => {
            res.json({
                status: 'healthy',
                model: 'llama3.2-vision:11b',
                service: 'Llama Vision API'
            });
        });

        // Image upload and analysis endpoint
        this.app.post('/api/analyze-image', this.upload.single('image'), async (req, res) => {
            try {
                if (!req.file) {
                    return res.status(400).json({ error: 'No image file provided' });
                }

                const { prompt } = req.body;
                const imagePath = req.file.path;

                // Read and encode image
                const imageBuffer = fs.readFileSync(imagePath);
                const imageBase64 = imageBuffer.toString('base64');

                // Call vision model
                const analysis = await this.callVisionModel(prompt, imageBase64);

                // Clean up uploaded file
                fs.unlinkSync(imagePath);

                res.json({
                    analysis,
                    filename: req.file.originalname,
                    fileSize: req.file.size,
                    model: 'llama3.2-vision:11b'
                });
            } catch (error) {
                res.status(500).json({ error: error.message });
            }
        });

        // Batch document processing
        this.app.post('/api/process-documents', this.upload.array('documents', 10), async (req, res) => {
            try {
                const documents = req.files;
                const results = [];

                for (const document of documents) {
                    const prompt = `
                    Extract and summarize the key information from this document.
                    Focus on main topics, key decisions, dates, and important data points.
                    Provide a structured summary with clear sections.
                    `;

                    const imageBuffer = fs.readFileSync(document.path);
                    const imageBase64 = imageBuffer.toString('base64');

                    const result = await this.callVisionModel(prompt, imageBase64);

                    results.push({
                        filename: document.originalname,
                        analysis: result,
                        fileSize: document.size
                    });

                    // Clean up
                    fs.unlinkSync(document.path);
                }

                res.json({
                    results,
                    processed: results.length,
                    model: 'llama3.2-vision:11b'
                });
            } catch (error) {
                res.status(500).json({ error: error.message });
            }
        });

        // Image comparison endpoint
        this.app.post('/api/compare-images', this.upload.array('images', 2), async (req, res) => {
            try {
                const images = req.files;
                if (images.length !== 2) {
                    return res.status(400).json({ error: 'Exactly 2 images required for comparison' });
                }

                const [image1, image2] = images;
                const prompt = "Compare these two images and identify all similarities and differences in detail.";

                // Read and encode both images
                const image1Buffer = fs.readFileSync(image1.path);
                const image2Buffer = fs.readFileSync(image2.path);
                const image1Base64 = image1Buffer.toString('base64');
                const image2Base64 = image2Buffer.toString('base64');

                const comparison = await this.callVisionModel(prompt, [image1Base64, image2Base64]);

                // Clean up
                fs.unlinkSync(image1.path);
                fs.unlinkSync(image2.path);

                res.json({
                    comparison,
                    image1: image1.originalname,
                    image2: image2.originalname,
                    model: 'llama3.2-vision:11b'
                });
            } catch (error) {
                res.status(500).json({ error: error.message });
            }
        });

        // Streaming image analysis for real-time applications
        this.app.post('/api/stream-analysis', this.upload.single('image'), (req, res) => {
            try {
                if (!req.file) {
                    return res.status(400).json({ error: 'No image file provided' });
                }

                const { prompt } = req.body;
                const imagePath = req.file.path;

                res.setHeader('Content-Type', 'text/event-stream');
                res.setHeader('Cache-Control', 'no-cache');
                res.setHeader('Connection', 'keep-alive');

                // Read and encode image
                const imageBuffer = fs.readFileSync(imagePath);
                const imageBase64 = imageBuffer.toString('base64');

                // Stream vision model response
                this.streamVisionResponse(prompt, imageBase64, res);
            } catch (error) {
                res.status(500).json({ error: error.message });
            }
        });
    }

    async callVisionModel(prompt, imageData) {
        const url = "http://localhost:11434/api/generate";

        const data = {
            model: "llama3.2-vision:11b",
            prompt: prompt,
            images: imageData,
            stream: false,
            options: {
                temperature: 0.7,
                top_p: 0.9
            }
        };

        const response = await fetch(url, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify(data)
        });

        if (!response.ok) {
            throw new Error(`Vision model error: ${response.status} ${response.statusText}`);
        }

        const result = await response.json();
        return result.response;
    }

    async streamVisionResponse(prompt, imageData, res) {
        const url = "http://localhost:11434/api/generate";

        const data = {
            model: "llama3.2-vision:11b",
            prompt: prompt,
            images: imageData,
            stream: true
        };

        const response = await fetch(url, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify(data)
        });

        if (!response.ok) {
            res.write(`data: ${JSON.stringify({ error: `API error: ${response.status}`})}

`);
            return res.end();
        }

        const reader = response.body.getReader();
        const decoder = new TextDecoder();

        try {
            while (true) {
                    const { done, value } = await reader.read();
                    if (done) break;

                    const chunk = decoder.decode(value, { stream: true });
                    if (chunk.includes('response')) {
                        const responseMatch = chunk.match(/"response":s*"([^"]+)"/);
                        if (responseMatch) {
                            res.write(`data: ${JSON.stringify({ response: responseMatch[1] })}

`);
                        }
                    }
                }
        } catch (error) {
            res.write(`data: ${JSON.stringify({ error: `Stream error: ${error.message}` })}

`);
        } finally {
            res.end();
        }
    }

    start() {
        this.app.listen(this.port, () => {
            console.log(`Llama 3.2 11B Vision Server running on port ${this.port}`);
        });
    }
}

// Initialize and start server
const server = new LlamaVisionServer();
server.start();