openai

/

clip-vit-base-patch32

like 458

I try to calculate image embeddings using Chromium and Javascript and run into error Error in `parameters.candidate_labels`: field required

Here is my code. What is wrong with it?

const API_KEY = 'myAccessToken'; // tried both of read and write with same error
const imageUrl = document.querySelector('img').src; // Get the image URL from the page

// Function to get the image as a base64 string
async function getImageBase64(imageUrl) {
    try {
        const response = await fetch(imageUrl);
        if (!response.ok) {
            throw new Error(`Network response was not ok: ${response.statusText}`);
        }
        const blob = await response.blob();
        return new Promise((resolve, reject) => {
            const reader = new FileReader();
            reader.onloadend = () => resolve(reader.result);
            reader.onerror = reject;
            reader.readAsDataURL(blob);
        });
    } catch (error) {
        throw new Error(`Failed to fetch image: ${error.message}`);
    }
}

// Function to get embeddings from Hugging Face CLIP model
async function getCLIPEmbeddings(imageBase64, retries = 3) {
    const API_URL = 'https://api-inference.huggingface.co/models/openai/clip-vit-base-patch32';

    const payload = {
        inputs: {
            image: imageBase64.split(',')[1], // Remove the data URL part
            candidate_labels: ["placeholder"] // Required field to satisfy the API requirement
        }
    };

    try {
        const response = await fetch(API_URL, {
            method: 'POST',
            headers: {
                'Authorization': `Bearer ${API_KEY}`,
                'Content-Type': 'application/json'
            },
            body: JSON.stringify(payload)
        });
        if (!response.ok) {
            const errorText = await response.text();
            const errorData = JSON.parse(errorText);
            if (response.status === 503 && retries > 0) {
                const waitTime = errorData.estimated_time * 1000; // Convert to milliseconds
                console.log(`Model loading, retrying in ${waitTime / 1000} seconds...`);
                await new Promise(resolve => setTimeout(resolve, waitTime));
                return getCLIPEmbeddings(imageBase64, retries - 1);
            }
            throw new Error(`API response was not ok: ${response.status} ${response.statusText} ${errorText}`);
        }
        const data = await response.json();
        return data;
    } catch (error) {
        throw new Error(`Failed to fetch embeddings: ${error.message}`);
    }
}

// Function to process the embedding and output to Screaming Frog
async function processImageEmbedding() {
    try {
        const imageBase64 = await getImageBase64(imageUrl);
        const embeddings = await getCLIPEmbeddings(imageBase64);
        console.log(embeddings); // Ensure this outputs the correct embeddings
        return embeddings.toString();
    } catch (error) {
        console.error('Error in processImageEmbedding:', error);
        throw error;
    }
}

return processImageEmbedding()
    .then(embeddings => seoSpider.data(embeddings))
    .catch(error => seoSpider.error(error));