diff --git a/CHANGELOG.md b/CHANGELOG.md index 55fc7eb..5eded45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ - The format is based on [Keep a Changelog](https://keepachangelog.com/). - This project adheres to [Semantic Versioning](https://semver.org/). +## Version 1.2.0 - tbd + +### Added + +- Support `VECTOR_EMBEDDING` function using ONNX Runtime with `Xenova/all-MiniLM-L6-v2` model (384 dimensions) + - Downloads model on-demand from Hugging Face (~10MB, cached locally) + - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` + - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions + - Synchronous execution suitable for SQLite user-defined functions + - **Note**: Produces 384-dimensional vectors (vs. 768 in SAP HANA) for efficiency in local development scenarios + + ## Version 1.1.0 - 2026-07-20 diff --git a/README.md b/README.md index 7a2de53..f162cf5 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,48 @@ resources: type: org.cloudfoundry.managed-service ``` +### 3. Vector Embedding API for Plugin Integration + +The `@cap-js/ai` plugin exports a standalone vector embedding function that can be used by other plugins (like `@cap-js/sqlite`) to generate embeddings using an ONNX model. + +#### Usage + +```javascript +import { vector_embedding } from '@cap-js/ai/vector-embedding'; + +// Model initializes automatically on import - just use it +const embeddingJSON = vector_embedding('Hello world', 'DOCUMENT', 'SAP_GXY.20250407'); +const embedding = JSON.parse(embeddingJSON); // Array of 384 float values +``` + +#### Function Signature + +```typescript +function vector_embedding( + text: string | null, + text_type: string, + model_and_version: string +): string +``` + +**Parameters:** +- `text` - Text to embed (returns zero vector if null or empty) +- `text_type` - Type of text, e.g., `'DOCUMENT'` (currently informational) +- `model_and_version` - Model identifier, e.g., `'SAP_GXY.20250407'` or `'SAP_GXY.20240715'` + +**Returns:** +- JSON stringified array of embedding values (384 dimensions) + +**Features:** +- **Auto-initialization**: ONNX model loads automatically when module is imported (top-level await) +- **Deterministic**: Same input always produces same output +- **Normalized vectors**: All embeddings are L2-normalized +- **Semantic similarity**: Embeddings capture text meaning for similarity search + +**Error Handling:** +- Throws if ONNX model failed to load during import +- Throws if embedding generation fails +- Import errors can be caught to detect if AI plugin is available ## Test the plugin locally diff --git a/lib/vector_embedding/InferenceSession.js b/lib/vector_embedding/InferenceSession.js new file mode 100644 index 0000000..5d52e09 --- /dev/null +++ b/lib/vector_embedding/InferenceSession.js @@ -0,0 +1,249 @@ +// Copy from onnxruntime-common/dist/cjs/inference-session-impl.js and referenced files by it +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// Adjusted to meet the needs of SQLite by making the run functions synchronous to avoid WorkerThreads +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); +const ort = require('onnxruntime-common'); +const binding = require('onnxruntime-node/dist/binding.js'); + +class InferenceSession { + constructor(handler) { + this.handler = handler; + } + + run(feeds) { + const fetches = {}; + let options = {}; + // check inputs + if ( + typeof feeds !== 'object' || + feeds === null || + feeds instanceof ort.Tensor || + Array.isArray(feeds) + ) { + throw new TypeError( + "'feeds' must be an object that use input names as keys and OnnxValue as corresponding values." + ); + } + // check if all inputs are in feed + for (const name of this.handler.inputNames) { + if (typeof feeds[name] === 'undefined') + throw new Error(`input '${name}' is missing in 'feeds'.`); + } + // if no fetches is specified, we use the full output names list + for (const name of this.handler.outputNames) { + fetches[name] = null; + } + // feeds, fetches and options are prepared + const results = this.handler.run(feeds, fetches, options); + const returnValue = {}; + for (const key in results) { + if (Object.hasOwnProperty.call(results, key)) { + const result = results[key]; + if (result instanceof ort.Tensor) returnValue[key] = result; + else returnValue[key] = new ort.Tensor(result.type, result.data, result.dims); + } + } + return returnValue; + } + + static async create(arg0) { + let filePathOrUint8Array; + if (arg0 instanceof Uint8Array) filePathOrUint8Array = arg0; + else + throw Error( + 'Argument is not supported. Check original InferenceSession implementation if this adjustment needs to be adopted' + ); + + // resolve backend, update session options with validated EPs, and create session handler + const [backend, optionsWithValidatedEPs] = await resolveBackendAndExecutionProviders(); + const handler = await backend.createInferenceSessionHandler( + filePathOrUint8Array, + optionsWithValidatedEPs + ); + return new InferenceSession(handler); + } +} + +// Copy from onnxruntime-common/dist/cjs/backend-impl.js +async function resolveBackendAndExecutionProviders() { + const backends = new Map(); + const backendsList = listSupportedBackends(); + for (const backend of backendsList) { + backends.set(backend.name, { backend: onnxruntimeBackend }); + } + const backendNames = [...backends.keys()]; + // try to resolve and initialize all requested backends + let backend; + const errors = []; + const availableBackendNames = new Set(); + for (const backendName of backendNames) { + // eslint-disable-next-line no-await-in-loop + const resolveResult = await tryResolveAndInitializeBackend(backendName, backends); + if (typeof resolveResult === 'string') { + errors.push({ name: backendName, err: resolveResult }); + } else { + if (!backend) { + backend = resolveResult; + } + if (backend === resolveResult) { + availableBackendNames.add(backendName); + } + } + } + // if no backend is available, throw error. + if (!backend) { + throw new Error( + `no available backend found. ERR: ${errors.map((e) => `[${e.name}] ${e.err}`).join(', ')}` + ); + } + return [ + backend, + new Proxy( + {}, + { + get: (target, prop) => { + if (prop === 'executionProviders') { + return []; + } + return Reflect.get(target, prop); + } + } + ) + ]; +} + +async function tryResolveAndInitializeBackend(backendName, backends) { + const backendInfo = backends.get(backendName); + if (!backendInfo) { + return 'backend not found.'; + } + if (backendInfo.initialized) { + return backendInfo.backend; + } else if (backendInfo.aborted) { + return backendInfo.error; + } else { + const isInitializing = !!backendInfo.initPromise; + try { + if (!isInitializing) { + backendInfo.initPromise = backendInfo.backend.init(backendName); + } + await backendInfo.initPromise; + backendInfo.initialized = true; + return backendInfo.backend; + } catch (e) { + if (!isInitializing) { + backendInfo.error = `${e}`; + backendInfo.aborted = true; + } + return backendInfo.error; + } finally { + delete backendInfo.initPromise; + } + } +} + +// Copy from test/bookshop/node_modules/onnxruntime-node/dist/backend.js +const dataTypeStrings = [ + undefined, + 'float32', + 'uint8', + 'int8', + 'uint16', + 'int16', + 'int32', + 'int64', + 'string', + 'bool', + 'float16', + 'float64', + 'uint32', + 'uint64', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'uint4', + 'int4' +]; +class OnnxruntimeSessionHandler { + static inferenceSession = new WeakMap(); + constructor(pathOrBuffer, options) { + binding.initOrt(); + OnnxruntimeSessionHandler.inferenceSession.set(this, new binding.binding.InferenceSession()); + if (typeof pathOrBuffer === 'string') { + OnnxruntimeSessionHandler.inferenceSession.get(this).loadModel(pathOrBuffer, options); + } else { + OnnxruntimeSessionHandler.inferenceSession + .get(this) + .loadModel(pathOrBuffer.buffer, pathOrBuffer.byteOffset, pathOrBuffer.byteLength, options); + } + // prepare input/output names and metadata + this.inputNames = []; + this.outputNames = []; + this.inputMetadata = []; + this.outputMetadata = []; + // this function takes raw metadata from binding and returns a tuple of the following 2 items: + // - an array of string representing names + // - an array of converted InferenceSession.ValueMetadata + const fillNamesAndMetadata = (rawMetadata) => { + const names = []; + const metadata = []; + for (const m of rawMetadata) { + names.push(m.name); + if (!m.isTensor) { + metadata.push({ name: m.name, isTensor: false }); + } else { + const type = dataTypeStrings[m.type]; + if (type === undefined) { + throw new Error(`Unsupported data type: ${m.type}`); + } + const shape = []; + for (let i = 0; i < m.shape.length; ++i) { + const dim = m.shape[i]; + if (dim === -1) { + shape.push(m.symbolicDimensions[i]); + } else if (dim >= 0) { + shape.push(dim); + } else { + throw new Error(`Invalid dimension: ${dim}`); + } + } + metadata.push({ + name: m.name, + isTensor: m.isTensor, + type, + shape + }); + } + } + return [names, metadata]; + }; + [this.inputNames, this.inputMetadata] = fillNamesAndMetadata( + OnnxruntimeSessionHandler.inferenceSession.get(this).inputMetadata + ); + [this.outputNames, this.outputMetadata] = fillNamesAndMetadata( + OnnxruntimeSessionHandler.inferenceSession.get(this).outputMetadata + ); + } + async dispose() { + OnnxruntimeSessionHandler.inferenceSession.get(this).dispose(); + } + run(feeds, fetches, options) { + return OnnxruntimeSessionHandler.inferenceSession.get(this).run(feeds, fetches, options); + } +} +class OnnxruntimeBackend { + init() {} + createInferenceSessionHandler(pathOrBuffer, options) { + return new OnnxruntimeSessionHandler(pathOrBuffer, options || {}); + } +} +const onnxruntimeBackend = new OnnxruntimeBackend(); +const listSupportedBackends = binding.binding.listSupportedBackends; + +export { InferenceSession }; diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js new file mode 100644 index 0000000..8a9402b --- /dev/null +++ b/lib/vector_embedding/embedding.js @@ -0,0 +1,207 @@ +import os from 'os'; +import path from 'path'; +import ort from 'onnxruntime-node'; +import { + downloadModelIfNeeded, + forceRedownloadModel, + loadModelAndVocab, + preTokenize, + wordPieceTokenize, + validateTokenIds +} from './model-utils.js'; + +const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'; +const MODEL_DIR = path.join(getDataDir(), 'models', MODEL_NAME.replace('/', '_')); +const FILES = ['onnx/model.onnx', 'tokenizer.json', 'tokenizer_config.json']; + +async function initializeModelAndVocab() { + try { + const result = await loadModelAndVocab(MODEL_DIR); + session = result.session; + vocab = result.vocab; + } catch { + await forceRedownloadModel(MODEL_DIR, FILES); + await downloadModelIfNeeded(MODEL_DIR, FILES, MODEL_NAME); + const result = await loadModelAndVocab(MODEL_DIR); + session = result.session; + vocab = result.vocab; + } +} + +/** + * Main tokenization function that combines all steps + */ +function wordPieceTokenizer(text, vocab, maxLength = 512) { + const unkToken = '[UNK]'; + const clsToken = '[CLS]'; + const sepToken = '[SEP]'; + + const clsId = vocab.get(clsToken) ?? 101; + const sepId = vocab.get(sepToken) ?? 102; + const unkId = vocab.get(unkToken) ?? 100; + + if (typeof clsId !== 'number' || typeof sepId !== 'number' || typeof unkId !== 'number') { + throw new Error('Special tokens must have numeric IDs'); + } + + const preTokens = preTokenize(text); + + const tokens = [clsToken]; + const ids = [clsId]; + + for (const preToken of preTokens) { + const lowercaseToken = preToken.toLowerCase(); + const wordPieceTokens = wordPieceTokenize(lowercaseToken, vocab, unkToken); + + for (const wpToken of wordPieceTokens) { + const tokenId = vocab.get(wpToken) ?? unkId; + tokens.push(wpToken); + ids.push(tokenId); + } + } + + tokens.push(sepToken); + ids.push(sepId); + + if (tokens.length <= maxLength) return [{ tokens, ids }]; + + // For longer texts, create overlapping chunks + const maxContentLength = maxLength - 2; + const overlap = Math.floor(maxContentLength * 0.1); + const chunkSize = maxContentLength - overlap; + + const chunks = []; + const contentTokens = tokens.slice(1, -1); + const contentIds = ids.slice(1, -1); + + for (let i = 0; i < contentTokens.length; i += chunkSize) { + const chunkTokens = [clsToken, ...contentTokens.slice(i, i + maxContentLength - 1), sepToken]; + const chunkIds = [clsId, ...contentIds.slice(i, i + maxContentLength - 1), sepId]; + + chunks.push({ + tokens: chunkTokens, + ids: chunkIds + }); + } + + return chunks; +} + +/** + * Process embeddings for multiple chunks and combine them + */ +function processChunkedEmbeddings(chunks, session) { + const embeddings = []; + + for (const chunk of chunks) { + const { ids } = chunk; + const validIds = validateTokenIds(ids); + + const inputIds = new BigInt64Array(validIds.map((i) => BigInt(i))); + const attentionMask = new BigInt64Array(validIds.length).fill(BigInt(1)); + const tokenTypeIds = new BigInt64Array(validIds.length).fill(BigInt(0)); + + const inputTensor = new ort.Tensor('int64', inputIds, [1, validIds.length]); + const attentionTensor = new ort.Tensor('int64', attentionMask, [1, validIds.length]); + const tokenTypeTensor = new ort.Tensor('int64', tokenTypeIds, [1, validIds.length]); + + const feeds = { + input_ids: inputTensor, + attention_mask: attentionTensor, + token_type_ids: tokenTypeTensor + }; + + const results = session.run(feeds); + const lastHiddenState = results['last_hidden_state']; + if (!lastHiddenState) + throw new Error( + `ONNX model output 'last_hidden_state' not found. Available outputs: ${Object.keys(results).join(', ')}` + ); + const [, sequenceLength, hiddenSize] = lastHiddenState.dims; + const embeddingData = lastHiddenState.data; + + // Apply mean pooling across the sequence dimension + const pooledEmbedding = new Float32Array(hiddenSize); + for (let i = 0; i < hiddenSize; i++) { + let sum = 0; + for (let j = 0; j < sequenceLength; j++) { + sum += embeddingData[j * hiddenSize + i]; + } + pooledEmbedding[i] = sum / sequenceLength; + } + + embeddings.push(pooledEmbedding); + } + + // If multiple chunks, average the embeddings + if (embeddings.length === 1) return embeddings[0]; + + const hiddenSize = embeddings[0].length; + const avgEmbedding = new Float32Array(hiddenSize); + + for (let i = 0; i < hiddenSize; i++) { + let sum = 0; + for (const embedding of embeddings) { + sum += embedding[i]; + } + avgEmbedding[i] = sum / embeddings.length; + } + + return avgEmbedding; +} + +let session = null; +let vocab = null; + +async function createSession() { + await downloadModelIfNeeded(MODEL_DIR, FILES, MODEL_NAME); + await initializeModelAndVocab(); +} + +function embedding(text) { + if (!session || !vocab) + throw new Error( + 'Embedding session not initialized. Call createSession() before using embedding().' + ); + const chunks = wordPieceTokenizer(text, vocab); + const vector = normalizeEmbedding(processChunkedEmbeddings(chunks, session)); + + const chunkObj = { content: text }; + return Object.defineProperty(chunkObj, 'embedding', { + value: vector, + writable: true, + configurable: true, + enumerable: false + }); + + function normalizeEmbedding(embedding) { + let norm = 0; + for (let i = 0; i < embedding.length; i++) { + norm += embedding[i] * embedding[i]; + } + norm = Math.sqrt(norm); + if (norm === 0) return embedding; // Guard against division by zero + for (let i = 0; i < embedding.length; i++) { + embedding[i] = embedding[i] / norm; + } + return embedding; + } +} + +/** + * Get the platform-specific data directory for the application + * @param {string} appName - The application name (defaults to 'semantic-search') + * @returns {string} The full path to the data directory + */ +function getDataDir(appName = 'semantic-search') { + const home = os.homedir(); + const dir = + os.platform() === 'win32' + ? process.env.LOCALAPPDATA || process.env.APPDATA || path.join(home, 'AppData', 'Local') + : process.env.XDG_DATA_HOME || path.join(home, '.local', 'share'); + + return path.join(dir, appName); +} + +export default embedding; +export { embedding, createSession }; diff --git a/lib/vector_embedding/index.js b/lib/vector_embedding/index.js new file mode 100644 index 0000000..6111447 --- /dev/null +++ b/lib/vector_embedding/index.js @@ -0,0 +1,49 @@ +import cds from '@sap/cds'; + +const LOG = cds.log('@cap-js/ai'); + +// Auto-initialize on module load +let embeddingModule; +let initializationError; + +try { + embeddingModule = await import('./embedding.js'); + await embeddingModule.createSession(); + LOG?.info?.('Vector embedding ONNX model initialized'); +} catch (err) { + LOG.warn('Failed to initialize embedding model:', err.message); + initializationError = err; +} + +const model_dimensions = { + 'SAP_GXY.20250407': 384, + 'SAP_GXY.20240715': 384 +}; + +/** + * Synchronous wrapper for vector embedding function. + * Generates embeddings using ONNX model. + * The model is initialized automatically when this module is imported. + * + * @param {string} text - Text to embed + * @param {string} text_type - Type of text (e.g., 'DOCUMENT') + * @param {string} model_and_version - Model identifier (e.g., 'SAP_GXY.20250407') + * @returns {string} JSON stringified array of embedding values + * @throws {Error} If embedding module failed to initialize or generation fails + */ +function vector_embedding(text, text_type, model_and_version) { + if (initializationError) { + throw new Error(`Embedding module failed to initialize: ${initializationError.message}`); + } + + if (!embeddingModule) { + throw new Error('Embedding module not available'); + } + + if (text) { + return JSON.stringify(Array.from(embeddingModule.embedding(text).embedding)); + } + return JSON.stringify(new Array(model_dimensions[model_and_version] ?? 384).fill(0)); +} + +export { vector_embedding }; diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js new file mode 100644 index 0000000..da9ef4e --- /dev/null +++ b/lib/vector_embedding/model-utils.js @@ -0,0 +1,123 @@ +import { InferenceSession } from './InferenceSession.js'; +import fs from 'fs/promises'; +import { constants } from 'fs'; +import path from 'path'; + +// File operations +async function fileExists(filePath) { + try { + await fs.access(filePath, constants.F_OK); + return true; + } catch { + return false; + } +} + +async function downloadFile(url, outputPath) { + const res = await fetch(url); + if (!res.ok) + throw new Error(`Failed to download ${url}, status ${res.status} (${res.statusText})`); + const arrayBuffer = await res.arrayBuffer(); + await fs.writeFile(outputPath, Buffer.from(arrayBuffer)); +} + +// Model management +async function downloadModelIfNeeded(modelDir, files, modelName) { + await fs.mkdir(modelDir, { recursive: true }); + // eslint-disable-next-line no-await-in-loop + for (const file of files) { + const filePath = path.join(modelDir, path.basename(file)); + // eslint-disable-next-line no-await-in-loop + if (!(await fileExists(filePath))) + // eslint-disable-next-line no-await-in-loop + await downloadFile(`https://huggingface.co/${modelName}/resolve/main/${file}`, filePath); + } +} + +async function forceRedownloadModel(modelDir, files) { + // eslint-disable-next-line no-await-in-loop + for (const file of files) { + const filePath = path.join(modelDir, path.basename(file)); + // eslint-disable-next-line no-await-in-loop + if (await fileExists(filePath)) await fs.unlink(filePath).catch(() => {}); + } +} + +async function loadModelAndVocab(modelDir) { + const modelPath = path.join(modelDir, 'model.onnx'); + const vocabPath = path.join(modelDir, 'tokenizer.json'); + + const session = await InferenceSession.create(await fs.readFile(modelPath)); + const tokenizerJson = JSON.parse(await fs.readFile(vocabPath, 'utf-8')); + + if (!tokenizerJson.model || !tokenizerJson.model.vocab) + throw new Error('Invalid tokenizer structure: missing model.vocab'); + + const cleanVocab = new Map(); + for (const [token, id] of Object.entries(tokenizerJson.model.vocab)) { + if (typeof id === 'number') cleanVocab.set(token, id); + } + + return { session, vocab: cleanVocab }; +} + +// Tokenization helpers +function preTokenize(text) { + return ( + text + .normalize('NFD') + // eslint-disable-next-line no-control-regex + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/g, '') + .replace(/\s+/g, ' ') + .trim() + .replace(/[!\s]\p{P}[!\s]/gu, (p) => ` ${p} `) + .split(/\s/g) + .filter((a) => a) + ); +} + +function wordPieceTokenize(token, vocab, unkToken = '[UNK]', maxInputCharsPerWord = 200) { + if (token.length > maxInputCharsPerWord) return [unkToken]; + + const outputTokens = []; + let start = 0; + while (start < token.length) { + let end = token.length; + let currentSubstring = null; + + while (start < end) { + let substring = token.substring(start, end); + if (start > 0) substring = '##' + substring; + if (vocab.has(substring)) { + currentSubstring = substring; + break; + } + end -= 1; + } + + if (currentSubstring === null) return [unkToken]; + + outputTokens.push(currentSubstring); + start = end; + } + + return outputTokens; +} + +// Validate token IDs before conversion to BigInt +function validateTokenIds(ids) { + ids.forEach((id) => { + if (typeof id !== 'number' || isNaN(id) || !isFinite(id)) + throw new Error(`Invalid token ID detected: ${id} (type: ${typeof id})`); + }); + return ids; +} + +export { + downloadModelIfNeeded, + forceRedownloadModel, + loadModelAndVocab, + preTokenize, + wordPieceTokenize, + validateTokenIds +}; diff --git a/package.json b/package.json index a16b543..78ebae7 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,10 @@ "author": "SAP SE (https://www.sap.com)", "homepage": "https://cap.cloud.sap/", "main": "cds-plugin.js", + "exports": { + ".": "./cds-plugin.js", + "./vector_embedding": "./lib/vector_embedding/index.js" + }, "scripts": { "lint": "npx -y eslint@10 .", "test": "node --test tests/*.test.js", @@ -20,6 +24,9 @@ "lib", "srv" ], + "dependencies": { + "onnxruntime-node": "^1.20.1" + }, "devDependencies": { "@cap-js/cds-test": "^1", "@cap-js/cds-types": "^0.16.0" diff --git a/tests/vector.test.js b/tests/vector.test.js new file mode 100644 index 0000000..4fb5257 --- /dev/null +++ b/tests/vector.test.js @@ -0,0 +1,118 @@ +import { describe, test } from 'node:test'; +import assert from 'node:assert'; +import { vector_embedding } from '../lib/vector_embedding/index.js'; + +describe('Vector embedding function (standalone)', () => { + describe('vector_embedding', () => { + test('computes embedding with ONNX model', async () => { + const result = vector_embedding('Hello world', 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Embedding should be an array'); + assert.strictEqual(embedding.length, 384, 'Embedding should have 384 dimensions'); + + // Check that values are floats in reasonable range + embedding.forEach((val, idx) => { + assert.strictEqual(typeof val, 'number', `Value at index ${idx} should be a number`); + assert.ok(Math.abs(val) <= 1, `Value at index ${idx} should be normalized (-1 to 1)`); + }); + }); + + test('deterministic - same input produces same output', async () => { + const e1 = vector_embedding('test text', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('test text', 'DOCUMENT', 'SAP_GXY.20250407'); + + assert.strictEqual(e1, e2, 'Same input should produce identical embeddings'); + }); + + test('different inputs produce different outputs', async () => { + const e1 = vector_embedding('hello world', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('goodbye world', 'DOCUMENT', 'SAP_GXY.20250407'); + + assert.notStrictEqual(e1, e2, 'Different inputs should produce different embeddings'); + }); + + test('semantically similar sentences produce similar vectors', async () => { + const e1 = vector_embedding('I love programming', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('I enjoy coding', 'DOCUMENT', 'SAP_GXY.20250407'); + + const v1 = JSON.parse(e1); + const v2 = JSON.parse(e2); + + const similarity = cosineSimilarity(v1, v2); + assert.ok( + similarity > 0.8, + `Semantically similar sentences should have high cosine similarity (got ${similarity.toFixed(3)})` + ); + }); + + test('semantically different sentences are far apart in vector space', async () => { + const e1 = vector_embedding('The cat sat on the mat', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('Quantum physics is fascinating', 'DOCUMENT', 'SAP_GXY.20250407'); + + const v1 = JSON.parse(e1); + const v2 = JSON.parse(e2); + + const similarity = cosineSimilarity(v1, v2); + assert.ok( + similarity < 0.1, + `Semantically different sentences should have low cosine similarity (got ${similarity.toFixed(3)})` + ); + }); + + test('handles empty text', async () => { + const result = vector_embedding('', 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Empty text should return zero vector'); + assert.strictEqual(embedding.length, 384, 'Should have 384 dimensions'); + assert.ok( + embedding.every((v) => v === 0), + 'Empty text should return all zeros' + ); + }); + + test('handles null text', async () => { + const result = vector_embedding(null, 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Null text should return zero vector'); + assert.strictEqual(embedding.length, 384, 'Should have 384 dimensions'); + assert.ok( + embedding.every((v) => v === 0), + 'Null text should return all zeros' + ); + }); + + test('uses correct dimensions for different models', async () => { + const result1 = vector_embedding('test', 'DOCUMENT', 'SAP_GXY.20250407'); + const embedding1 = JSON.parse(result1); + assert.strictEqual(embedding1.length, 384, 'SAP_GXY.20250407 should have 384 dimensions'); + + const result2 = vector_embedding('test', 'DOCUMENT', 'SAP_GXY.20240715'); + const embedding2 = JSON.parse(result2); + assert.strictEqual(embedding2.length, 384, 'SAP_GXY.20240715 should have 384 dimensions'); + + const result3 = vector_embedding('test', 'DOCUMENT', 'unknown_model'); + const embedding3 = JSON.parse(result3); + assert.strictEqual(embedding3.length, 384, 'Unknown model should default to 384 dimensions'); + }); + }); +}); + +// Helper function to calculate cosine similarity between two vectors +function cosineSimilarity(a, b) { + if (a.length !== b.length) throw new Error('Vectors must have the same length'); + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +}