Failed to store embeddings in Milvus: Error: Insert fail: some field does not exist for this collection in line. 0

Question

Trying to add some embeddings to milvus bd collection but met with the above error.

Code:

const { MilvusClient, DataType } = require('@zilliz/milvus2-sdk-node');
const express = require('express');
const app = express();
app.use(express.json());

const client = new MilvusClient("localhost:19530");

let idCounter = 0;

async function main() {
try {
const collections = await client.showCollections();
console.log("List all collections:\n", collections);
} catch (error) {
console.error("Error listing collections:", error);
}
}

async function storeEmbeddings(filename, embeddings) {
try {
const collectionName = 'embedding_test_2';

    // Check if collection already exists
    const collectionsInfo = await client.showCollections();
    const existingCollections = collectionsInfo.collection_names || [];
    console.log('Existing collections:', existingCollections);

    if (!existingCollections.includes(collectionName)) {
        await client.createCollection({
            collection_name: collectionName,
            fields: [
                { name: "my_id", data_type: DataType.Int64, is_primary_key: true, auto_id: false }, // Ensure primary key is int64
                { name: "filename", data_type: DataType.VarChar, max_length: 256 }, // varchar type for strings
                { name: "embedding", data_type: DataType.FloatVector, dim: embeddings.length } // Correct field definition
            ],
        });
        console.log(`Collection ${collectionName} created`);
    } else {
        console.log(`Collection ${collectionName} already exists`);
    }

    // Logging field names and data to be inserted
    console.log('Inserting data:', {
        collection_name: collectionName,
        fields_data: [
            { name: "my_id", values: idCounter + 1 },
            { name: "filename", values: filename },
            { name: "embedding", values: [embeddings] }
        ]
    });

    // Log the embedding array
    console.log('Embedding array:', embeddings);

    await client.insert({
        collection_name: collectionName,
        fields_data: [
            { name: "my_id", values: [idCounter++] },
            { name: "filename", values: [filename] },
            { name: "embedding", values: [embeddings] }
        ]
    });

    console.log("Embeddings stored successfully in Milvus");
} catch (error) {
    console.error('Failed to store embeddings in Milvus:', error);
}
}

module.exports = {
storeEmbeddings,
main
};

error:

Failed to store embeddings in Milvus: Error: Insert fail: some field does not exist for this collection in line. 0
at /Users/razataiab/Desktop/custom-rag/node_modules/@zilliz/milvus2-sdk-node/dist/milvus/grpc/Data.js:195:31
at Array.forEach ()
at /Users/razataiab/Desktop/custom-rag/node_modules/@zilliz/milvus2-sdk-node/dist/milvus/grpc/Data.js:192:28
at Array.forEach ()
at MilvusClient. (/Users/razataiab/Desktop/custom-rag/node_modules/@zilliz/milvus2-sdk-node/dist/milvus/grpc/Data.js:184:30)
at Generator.next ()
at fulfilled (/Users/razataiab/Desktop/custom-rag/node_modules/@zilliz/milvus2-sdk-node/dist/milvus/grpc/Data.js:5:58)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)

Trying to add some embeddings to milvus bd collection but met with the above error.

Answer 1

you should prepare row based data here. like this:

await client.insert({
  collection_name: collectionName,
  data: [
    { filename: 'some-file-name1',  embedding: [0.1,0.2,...]},
    { filename: 'some-file-name1',  embedding: [0.1,0.2,...]},
  ]
});

For now, the Node sdk accepts row based file, thanks.