初始化

This commit is contained in:
2026-02-22 18:55:40 +08:00
commit 8392cdd861
496 changed files with 45020 additions and 0 deletions

View File

@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-aliyun</name>
<artifactId>easy-agents-store-aliyun</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,221 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.aliyun;
import com.easyagents.core.document.Document;
import com.easyagents.core.model.client.HttpClient;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.StringUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* 文档 https://help.aliyun.com/document_detail/2510317.html
*/
public class AliyunVectorStore extends DocumentStore {
private static final Logger LOG = LoggerFactory.getLogger(AliyunVectorStore.class);
private AliyunVectorStoreConfig config;
private final HttpClient httpUtil = new HttpClient();
public AliyunVectorStore(AliyunVectorStoreConfig config) {
this.config = config;
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("dashvector-auth-token", config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
List<Map<String, Object>> payloadDocs = new ArrayList<>();
for (Document vectorDocument : documents) {
Map<String, Object> document = new HashMap<>();
if (vectorDocument.getMetadataMap() != null) {
document.put("fields", vectorDocument.getMetadataMap());
}
document.put("vector", vectorDocument.getVector());
document.put("id", vectorDocument.getId());
payloadDocs.add(document);
}
payloadMap.put("docs", payloadDocs);
String payload = JSON.toJSONString(payloadMap);
String url = "https://" + config.getEndpoint() + "/v1/collections/"
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
String response = httpUtil.post(url, headers, payload);
if (StringUtil.noText(response)) {
return StoreResult.fail();
}
JSONObject jsonObject = JSON.parseObject(response);
Integer code = jsonObject.getInteger("code");
String message = jsonObject.getString("message");
if (code != null && code == 0 && "Success".equals(message)) {
return StoreResult.successWithIds(documents);
} else {
LOG.error("delete vector fail: " + response);
return StoreResult.fail(message);
}
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("dashvector-auth-token", config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("ids", ids);
String payload = JSON.toJSONString(payloadMap);
String url = "https://" + config.getEndpoint() + "/v1/collections/"
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
String response = httpUtil.delete(url, headers, payload);
if (StringUtil.noText(response)) {
return StoreResult.fail();
}
JSONObject jsonObject = JSON.parseObject(response);
Integer code = jsonObject.getInteger("code");
if (code != null && code == 0) {
return StoreResult.success();
} else {
LOG.error("delete vector fail: " + response);
return StoreResult.fail();
}
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("dashvector-auth-token", config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
List<Map<String, Object>> payloadDocs = new ArrayList<>();
for (Document vectorDocument : documents) {
Map<String, Object> document = new HashMap<>();
if (vectorDocument.getMetadataMap() != null) {
document.put("fields", vectorDocument.getMetadataMap());
}
document.put("vector", vectorDocument.getVector());
document.put("id", vectorDocument.getId());
payloadDocs.add(document);
}
payloadMap.put("docs", payloadDocs);
String payload = JSON.toJSONString(payloadMap);
String url = "https://" + config.getEndpoint() + "/v1/collections/"
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
String response = httpUtil.put(url, headers, payload);
if (StringUtil.noText(response)) {
return StoreResult.fail();
}
JSONObject jsonObject = JSON.parseObject(response);
Integer code = jsonObject.getInteger("code");
if (code != null && code == 0) {
return StoreResult.successWithIds(documents);
} else {
LOG.error("delete vector fail: " + response);
return StoreResult.fail();
}
}
@Override
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("dashvector-auth-token", config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("vector", wrapper.getVector());
payloadMap.put("topk", wrapper.getMaxResults());
payloadMap.put("include_vector", wrapper.isWithVector());
payloadMap.put("filter", wrapper.toFilterExpression());
String payload = JSON.toJSONString(payloadMap);
String url = "https://" + config.getEndpoint() + "/v1/collections/"
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/query";
String result = httpUtil.post(url, headers, payload);
if (StringUtil.noText(result)) {
return null;
}
//https://help.aliyun.com/document_detail/2510319.html
JSONObject rootObject = JSON.parseObject(result);
int code = rootObject.getIntValue("code");
if (code != 0) {
//error
LoggerFactory.getLogger(AliyunVectorStore.class).error("can not search data AliyunVectorStorecode: " + code + "), message: " + rootObject.getString("message"));
return null;
}
JSONArray output = rootObject.getJSONArray("output");
List<Document> documents = new ArrayList<>(output.size());
for (int i = 0; i < output.size(); i++) {
JSONObject jsonObject = output.getJSONObject(i);
Document document = new Document();
document.setId(jsonObject.getString("id"));
document.setVector(jsonObject.getObject("vector", float[].class));
// 阿里云数据采用余弦相似度计算 jsonObject.getDoubleValue("score") 表示余弦距离,
// 原始余弦距离范围是[0, 2]0表示最相似2表示最不相似
Double distance = jsonObject.getDouble("score");
if (distance != null) {
double score = distance / 2.0;
document.setScore(1.0d - score);
}
JSONObject fields = jsonObject.getJSONObject("fields");
document.addMetadata(fields);
documents.add(document);
}
return documents;
}
}

View File

@@ -0,0 +1,67 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.aliyun;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
/**
* https://help.aliyun.com/document_detail/2510317.html
*/
public class AliyunVectorStoreConfig implements DocumentStoreConfig {
private String endpoint;
private String apiKey;
private String database;
private String defaultCollectionName;
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.endpoint, this.apiKey, this.database, this.defaultCollectionName);
}
}

View File

@@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-chroma</name>
<artifactId>easy-agents-store-chroma</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
</dependency>
<!-- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.10.1</version>
</dependency> -->
<!-- 测试依赖 -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,101 @@
/*
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.chroma;
import com.easyagents.core.store.condition.Condition;
import com.easyagents.core.store.condition.ConditionType;
import com.easyagents.core.store.condition.ExpressionAdaptor;
import com.easyagents.core.store.condition.Value;
import java.util.StringJoiner;
public class ChromaExpressionAdaptor implements ExpressionAdaptor {
public static final ChromaExpressionAdaptor DEFAULT = new ChromaExpressionAdaptor();
@Override
public String toOperationSymbol(ConditionType type) {
if (type == ConditionType.EQ) {
return " == ";
} else if (type == ConditionType.NE) {
return " != ";
} else if (type == ConditionType.GT) {
return " > ";
} else if (type == ConditionType.GE) {
return " >= ";
} else if (type == ConditionType.LT) {
return " < ";
} else if (type == ConditionType.LE) {
return " <= ";
} else if (type == ConditionType.IN) {
return " IN ";
}
return type.getDefaultSymbol();
}
@Override
public String toCondition(Condition condition) {
if (condition.getType() == ConditionType.BETWEEN) {
Object[] values = (Object[]) ((Value) condition.getRight()).getValue();
return "(" + toLeft(condition.getLeft())
+ toOperationSymbol(ConditionType.GE)
+ values[0] + " && "
+ toLeft(condition.getLeft())
+ toOperationSymbol(ConditionType.LE)
+ values[1] + ")";
}
return ExpressionAdaptor.super.toCondition(condition);
}
@Override
public String toValue(Condition condition, Object value) {
if (value == null) {
return "null";
}
if (condition.getType() == ConditionType.IN) {
Object[] values = (Object[]) value;
StringJoiner stringJoiner = new StringJoiner(",", "[", "]");
for (Object v : values) {
if (v != null) {
stringJoiner.add("\"" + v + "\"");
}
}
return stringJoiner.toString();
} else if (value instanceof String) {
return "\"" + value + "\"";
} else if (value instanceof Boolean) {
return ((Boolean) value).toString();
} else if (value instanceof Number) {
return value.toString();
}
return ExpressionAdaptor.super.toValue(condition, value);
}
public String toLeft(Object left) {
if (left instanceof String) {
String field = (String) left;
if (field.contains(".")) {
return field;
}
return field;
}
return left.toString();
}
}

View File

@@ -0,0 +1,794 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.chroma;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.store.condition.ExpressionAdaptor;
import com.easyagents.core.model.client.HttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
/**
* ChromaVectorStore class provides an interface to interact with Chroma Vector Database
* using direct HTTP calls to the Chroma REST API.
*/
public class ChromaVectorStore extends DocumentStore {
private static final Logger logger = LoggerFactory.getLogger(ChromaVectorStore.class);
private final String baseUrl;
private final String collectionName;
private final String tenant;
private final String database;
private final ChromaVectorStoreConfig config;
private final ExpressionAdaptor expressionAdaptor;
private final HttpClient httpClient;
private final int MAX_RETRIES = 3;
private final long RETRY_INTERVAL_MS = 1000;
private static final String BASE_API = "/api/v2";
public ChromaVectorStore(ChromaVectorStoreConfig config) {
Objects.requireNonNull(config, "ChromaVectorStoreConfig cannot be null");
this.baseUrl = config.getBaseUrl();
this.tenant = config.getTenant();
this.database = config.getDatabase();
this.collectionName = config.getCollectionName();
this.config = config;
this.expressionAdaptor = ChromaExpressionAdaptor.DEFAULT;
// 创建并配置HttpClient实例
this.httpClient = createHttpClient();
// 验证配置的有效性
validateConfig();
// 如果配置了自动创建集合,检查并创建集合
if (config.isAutoCreateCollection()) {
try {
// 确保租户和数据库存在
ensureTenantAndDatabaseExists();
// 确保集合存在
ensureCollectionExists();
} catch (Exception e) {
logger.warn("Failed to ensure collection exists: {}. Will retry on first operation.", e.getMessage());
}
}
}
private HttpClient createHttpClient() {
HttpClient client = new HttpClient();
return client;
}
private void validateConfig() {
if (baseUrl == null || baseUrl.isEmpty()) {
throw new IllegalArgumentException("Base URL cannot be empty");
}
if (!baseUrl.startsWith("http://") && !baseUrl.startsWith("https://")) {
throw new IllegalArgumentException("Base URL must start with http:// or https://");
}
}
/**
* 确保租户和数据库存在,如果不存在则创建
*/
private void ensureTenantAndDatabaseExists() {
try {
// 检查并创建租户
if (tenant != null && !tenant.isEmpty()) {
ensureTenantExists();
// 检查并创建数据库(如果租户已设置)
if (database != null && !database.isEmpty()) {
ensureDatabaseExists();
}
}
} catch (Exception e) {
logger.error("Error ensuring tenant and database exist", e);
}
}
/**
* 确保租户存在,如果不存在则创建
*/
private void ensureTenantExists() throws IOException {
String tenantUrl = baseUrl + BASE_API + "/tenants/" + tenant;
Map<String, String> headers = createHeaders();
try {
// 尝试获取租户信息
String responseBody = executeWithRetry(() -> httpClient.get(tenantUrl, headers));
logger.debug("Successfully verified tenant '{}' exists", tenant);
} catch (IOException e) {
// 如果获取失败,尝试创建租户
logger.info("Creating tenant '{}' as it does not exist", tenant);
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("name", tenant);
String createTenantUrl = baseUrl + BASE_API + "/tenants";
String jsonRequestBody = safeJsonSerialize(requestBody);
String responseBody = executeWithRetry(() -> httpClient.post(createTenantUrl, headers, jsonRequestBody));
logger.info("Successfully created tenant '{}'", tenant);
}
}
/**
* 确保数据库存在,如果不存在则创建
*/
private void ensureDatabaseExists() throws IOException {
if (tenant == null || tenant.isEmpty()) {
throw new IllegalStateException("Cannot create database without tenant");
}
String databaseUrl = baseUrl + BASE_API + "/tenants/" + tenant + "/databases/" + database;
Map<String, String> headers = createHeaders();
try {
// 尝试获取数据库信息
String responseBody = executeWithRetry(() -> httpClient.get(databaseUrl, headers));
logger.debug("Successfully verified database '{}' exists in tenant '{}'",
database, tenant);
} catch (IOException e) {
// 如果获取失败,尝试创建数据库
logger.info("Creating database '{}' in tenant '{}' as it does not exist",
database, tenant);
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("name", database);
String createDatabaseUrl = baseUrl + BASE_API + "/tenants/" + tenant + "/databases";
String jsonRequestBody = safeJsonSerialize(requestBody);
String responseBody = executeWithRetry(() -> httpClient.post(createDatabaseUrl, headers, jsonRequestBody));
logger.info("Successfully created database '{}' in tenant '{}'",
database, tenant);
}
}
/**
* 根据collectionName查询Collection ID
*/
private String getCollectionId(String collectionName) throws IOException {
String collectionsUrl = buildCollectionsUrl();
Map<String, String> headers = createHeaders();
String responseBody = executeWithRetry(() -> httpClient.get(collectionsUrl, headers));
if (responseBody == null) {
throw new IOException("Failed to get collections, no response");
}
Object responseObj = parseJsonResponse(responseBody);
List<Map<String, Object>> collections = new ArrayList<>();
// 处理不同格式的响应
if (responseObj instanceof Map) {
Map<String, Object> responseMap = (Map<String, Object>) responseObj;
if (responseMap.containsKey("collections") && responseMap.get("collections") instanceof List) {
collections = (List<Map<String, Object>>) responseMap.get("collections");
}
} else if (responseObj instanceof List) {
List<?> rawCollections = (List<?>) responseObj;
for (Object item : rawCollections) {
if (item instanceof Map) {
collections.add((Map<String, Object>) item);
}
}
}
// 查找指定名称的集合
for (Map<String, Object> collection : collections) {
if (collection.containsKey("name") && collectionName.equals(collection.get("name"))) {
return collection.get("id").toString();
}
}
throw new IOException("Collection not found: " + collectionName);
}
private void createCollection() throws IOException {
// 构建创建集合的API URL包含tenant和database
String createCollectionUrl = buildCollectionsUrl();
Map<String, String> headers = createHeaders();
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("name", collectionName);
String jsonRequestBody = safeJsonSerialize(requestBody);
String responseBody = executeWithRetry(() -> httpClient.post(createCollectionUrl, headers, jsonRequestBody));
if (responseBody == null) {
throw new IOException("Failed to create collection: no response");
}
try {
Object responseObj = parseJsonResponse(responseBody);
Map<String, Object> responseMap = null;
if (responseObj instanceof Map) {
responseMap = (Map<String, Object>) responseObj;
}
if (responseMap.containsKey("error")) {
throw new IOException("Failed to create collection: " + responseMap.get("error"));
}
logger.info("Collection '{}' created successfully", collectionName);
} catch (Exception e) {
throw new IOException("Failed to process collection creation response: " + e.getMessage(), e);
}
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
Objects.requireNonNull(documents, "Documents cannot be null");
if (documents.isEmpty()) {
logger.debug("No documents to store");
return StoreResult.success();
}
try {
// 确保集合存在
ensureCollectionExists();
String collectionName = getCollectionName(options);
List<String> ids = new ArrayList<>();
List<List<Double>> embeddings = new ArrayList<>();
List<Map<String, Object>> metadatas = new ArrayList<>();
List<String> documentsContent = new ArrayList<>();
for (Document doc : documents) {
ids.add(String.valueOf(doc.getId()));
if (doc.getVector() != null) {
List<Double> embedding = doc.getVectorAsDoubleList();
embeddings.add(embedding);
} else {
embeddings.add(null);
}
Map<String, Object> metadata = doc.getMetadataMap() != null ?
new HashMap<>(doc.getMetadataMap()) : new HashMap<>();
metadatas.add(metadata);
documentsContent.add(doc.getContent());
}
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("ids", ids);
requestBody.put("embeddings", embeddings);
requestBody.put("metadatas", metadatas);
requestBody.put("documents", documentsContent);
String collectionId = getCollectionId(collectionName);
// 构建包含tenant和database的完整URL
String collectionUrl = buildCollectionUrl(collectionId, "add");
Map<String, String> headers = createHeaders();
String jsonRequestBody = safeJsonSerialize(requestBody);
logger.debug("Storing {} documents to collection '{}'", documents.size(), collectionName);
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
if (responseBody == null) {
logger.error("Error storing documents: no response");
return StoreResult.fail();
}
Object responseObj = parseJsonResponse(responseBody);
Map<String, Object> responseMap = null;
if (responseObj instanceof Map) {
responseMap = (Map<String, Object>) responseObj;
}
if (responseMap.containsKey("error")) {
String errorMsg = "Error storing documents: " + responseMap.get("error");
logger.error(errorMsg);
return StoreResult.fail();
}
logger.debug("Successfully stored {} documents", documents.size());
return StoreResult.successWithIds(documents);
} catch (Exception e) {
logger.error("Error storing documents to Chroma", e);
return StoreResult.fail();
}
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
Objects.requireNonNull(ids, "IDs cannot be null");
if (ids.isEmpty()) {
logger.debug("No IDs to delete");
return StoreResult.success();
}
try {
// 确保集合存在
ensureCollectionExists();
String collectionName = getCollectionName(options);
List<String> stringIds = ids.stream()
.map(Object::toString)
.collect(Collectors.toList());
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("ids", stringIds);
String collectionId = getCollectionId(collectionName);
// 构建包含tenant和database的完整URL
String collectionUrl = buildCollectionUrl(collectionId, "delete");
Map<String, String> headers = createHeaders();
String jsonRequestBody = safeJsonSerialize(requestBody);
logger.debug("Deleting {} documents from collection '{}'", ids.size(), collectionName);
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
if (responseBody == null) {
logger.error("Error deleting documents: no response");
return StoreResult.fail();
}
Object responseObj = parseJsonResponse(responseBody);
Map<String, Object> responseMap = null;
if (responseObj instanceof Map) {
responseMap = (Map<String, Object>) responseObj;
}
if (responseMap.containsKey("error")) {
String errorMsg = "Error deleting documents: " + responseMap.get("error");
logger.error(errorMsg);
return StoreResult.fail();
}
logger.debug("Successfully deleted {} documents", ids.size());
return StoreResult.success();
} catch (Exception e) {
logger.error("Error deleting documents from Chroma", e);
return StoreResult.fail();
}
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
Objects.requireNonNull(documents, "Documents cannot be null");
if (documents.isEmpty()) {
logger.debug("No documents to update");
return StoreResult.success();
}
try {
// Chroma doesn't support direct update, so we delete and re-add
List<Object> ids = documents.stream().map(Document::getId).collect(Collectors.toList());
StoreResult deleteResult = doDelete(ids, options);
if (!deleteResult.isSuccess()) {
logger.warn("Delete failed during update operation: {}", deleteResult.toString());
// 尝试继续添加,因为可能有些文档是新的
}
StoreResult storeResult = doStore(documents, options);
if (storeResult.isSuccess()) {
logger.debug("Successfully updated {} documents", documents.size());
}
return storeResult;
} catch (Exception e) {
logger.error("Error updating documents in Chroma", e);
return StoreResult.fail();
}
}
@Override
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
Objects.requireNonNull(wrapper, "SearchWrapper cannot be null");
try {
// 确保集合存在
ensureCollectionExists();
String collectionName = getCollectionName(options);
int limit = wrapper.getMaxResults() > 0 ? wrapper.getMaxResults() : 10;
Map<String, Object> requestBody = new HashMap<>();
// 检查查询条件是否有效
if (wrapper.getVector() == null && wrapper.getText() == null) {
throw new IllegalArgumentException("Either vector or text must be provided for search");
}
// 设置查询向量
if (wrapper.getVector() != null) {
List<Double> queryEmbedding = wrapper.getVectorAsDoubleList();
requestBody.put("query_embeddings", Collections.singletonList(queryEmbedding));
logger.debug("Performing vector search with dimension: {}", queryEmbedding.size());
} else if (wrapper.getText() != null) {
requestBody.put("query_texts", Collections.singletonList(wrapper.getText()));
logger.debug("Performing text search: {}", sanitizeLogString(wrapper.getText(), 100));
}
// 设置返回数量
requestBody.put("n_results", limit);
// 设置过滤条件
if (wrapper.getCondition() != null) {
try {
String whereClause = expressionAdaptor.toCondition(wrapper.getCondition());
// Chroma的where条件是JSON对象需要解析
Object whereObj = parseJsonResponse(whereClause);
Map<String, Object> whereMap = null;
if (whereObj instanceof Map) {
whereMap = (Map<String, Object>) whereObj;
}
requestBody.put("where", whereMap);
logger.debug("Search with filter condition: {}", whereClause);
} catch (Exception e) {
logger.warn("Failed to parse filter condition: {}, ignoring condition", e.getMessage());
}
}
String collectionId = getCollectionId(collectionName);
// 构建包含tenant和database的完整URL
String collectionUrl = buildCollectionUrl(collectionId, "query");
Map<String, String> headers = createHeaders();
String jsonRequestBody = safeJsonSerialize(requestBody);
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
if (responseBody == null) {
logger.error("Error searching documents: no response");
return Collections.emptyList();
}
Object responseObj = parseJsonResponse(responseBody);
Map<String, Object> responseMap = null;
if (responseObj instanceof Map) {
responseMap = (Map<String, Object>) responseObj;
}
// 检查响应是否包含error字段
if (responseMap.containsKey("error")) {
logger.error("Error searching documents: {}", responseMap.get("error"));
return Collections.emptyList();
}
// 解析结果
return parseSearchResults(responseMap);
} catch (Exception e) {
logger.error("Error searching documents in Chroma", e);
return Collections.emptyList();
}
}
/**
* 支持直接使用向量数组和topK参数的搜索方法
*/
public List<Document> searchInternal(double[] vector, int topK, StoreOptions options) {
Objects.requireNonNull(vector, "Vector cannot be null");
if (topK <= 0) {
topK = 10;
}
try {
// 确保集合存在
ensureCollectionExists();
String collectionName = getCollectionName(options);
Map<String, Object> requestBody = new HashMap<>();
// 设置查询向量
List<Double> queryEmbedding = Arrays.stream(vector)
.boxed()
.collect(Collectors.toList());
requestBody.put("query_embeddings", Collections.singletonList(queryEmbedding));
// 设置返回数量
requestBody.put("n_results", topK);
String collectionId = getCollectionId(collectionName);
// 构建包含tenant和database的完整URL
String collectionUrl = buildCollectionUrl(collectionId, "query");
Map<String, String> headers = createHeaders();
String jsonRequestBody = safeJsonSerialize(requestBody);
logger.debug("Performing direct vector search with dimension: {}", vector.length);
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
if (responseBody == null) {
logger.error("Error searching documents: no response");
return Collections.emptyList();
}
Object responseObj = parseJsonResponse(responseBody);
Map<String, Object> responseMap = null;
if (responseObj instanceof Map) {
responseMap = (Map<String, Object>) responseObj;
}
// 检查响应是否包含error字段
if (responseMap.containsKey("error")) {
logger.error("Error searching documents: {}", responseMap.get("error"));
return Collections.emptyList();
}
// 解析结果
return parseSearchResults(responseMap);
} catch (Exception e) {
logger.error("Error searching documents in Chroma", e);
return Collections.emptyList();
}
}
private List<Document> parseSearchResults(Map<String, Object> responseMap) {
try {
List<String> ids = extractResultsFromNestedList(responseMap, "ids");
List<String> documents = extractResultsFromNestedList(responseMap, "documents");
List<Map<String, Object>> metadatas = extractResultsFromNestedList(responseMap, "metadatas");
List<List<Double>> embeddings = extractResultsFromNestedList(responseMap, "embeddings");
List<Double> distances = extractResultsFromNestedList(responseMap, "distances");
if (ids == null || ids.isEmpty()) {
logger.debug("No documents found in search results");
return Collections.emptyList();
}
// 转换为Easy-Agents的Document格式
List<Document> resultDocs = new ArrayList<>();
for (int i = 0; i < ids.size(); i++) {
Document doc = new Document();
doc.setId(ids.get(i));
if (documents != null && i < documents.size()) {
doc.setContent(documents.get(i));
}
if (metadatas != null && i < metadatas.size()) {
doc.setMetadataMap(metadatas.get(i));
}
if (embeddings != null && i < embeddings.size() && embeddings.get(i) != null) {
doc.setVector(embeddings.get(i));
}
// 设置相似度分数(距离越小越相似)
if (distances != null && i < distances.size()) {
double score = 1.0 - distances.get(i);
// 确保分数在合理范围内
score = Math.max(0, Math.min(1, score));
doc.setScore(score);
}
resultDocs.add(doc);
}
logger.debug("Found {} documents in search results", resultDocs.size());
return resultDocs;
} catch (Exception e) {
logger.error("Failed to parse search results", e);
return Collections.emptyList();
}
}
@SuppressWarnings("unchecked")
private <T> List<T> extractResultsFromNestedList(Map<String, Object> responseMap, String key) {
try {
if (!responseMap.containsKey(key)) {
return null;
}
List<?> outerList = (List<?>) responseMap.get(key);
if (outerList == null || outerList.isEmpty()) {
return null;
}
// Chroma返回的结果是嵌套列表第一个元素是当前查询的结果
return (List<T>) outerList.get(0);
} catch (Exception e) {
logger.warn("Failed to extract '{}' from response: {}", key, e.getMessage());
return null;
}
}
private Map<String, String> createHeaders() {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
if (config.getApiKey() != null && !config.getApiKey().isEmpty()) {
headers.put("X-Chroma-Token", config.getApiKey());
}
// 添加租户和数据库信息(如果配置了)
if (tenant != null && !tenant.isEmpty()) {
headers.put("X-Chroma-Tenant", tenant);
}
if (database != null && !database.isEmpty()) {
headers.put("X-Chroma-Database", database);
}
return headers;
}
private <T> T executeWithRetry(HttpOperation<T> operation) throws IOException {
int attempts = 0;
IOException lastException = null;
while (attempts < MAX_RETRIES) {
try {
attempts++;
return operation.execute();
} catch (IOException e) {
lastException = e;
// 如果是最后一次尝试,则抛出异常
if (attempts >= MAX_RETRIES) {
throw new IOException("Operation failed after " + MAX_RETRIES + " attempts: " + e.getMessage(), e);
}
// 记录重试信息
logger.warn("Operation failed (attempt {} of {}), retrying in {}ms: {}",
attempts, MAX_RETRIES, RETRY_INTERVAL_MS, e.getMessage());
// 等待一段时间后重试
try {
Thread.sleep(RETRY_INTERVAL_MS);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("Retry interrupted", ie);
}
}
}
// 这一行理论上不会执行到,但为了编译器满意
throw lastException != null ? lastException : new IOException("Operation failed without exception");
}
private String safeJsonSerialize(Map<String, Object> map) {
// 使用标准的JSON序列化但在实际应用中可以添加更多的安全检查
try {
return new com.google.gson.Gson().toJson(map);
} catch (Exception e) {
throw new RuntimeException("Failed to serialize request body to JSON", e);
}
}
private Object parseJsonResponse(String json) {
try {
if (json == null || json.trim().isEmpty()) {
return null;
}
// Check if JSON starts with [ indicating an array
if (json.trim().startsWith("[")) {
return new com.google.gson.Gson().fromJson(json, List.class);
} else {
// Otherwise assume it's an object
return new com.google.gson.Gson().fromJson(json, Map.class);
}
} catch (Exception e) {
throw new RuntimeException("Failed to parse JSON response: " + json, e);
}
}
private String sanitizeLogString(String input, int maxLength) {
if (input == null) {
return null;
}
String sanitized = input.replaceAll("[\n\r]", " ");
return sanitized.length() > maxLength ? sanitized.substring(0, maxLength) + "..." : sanitized;
}
private String getCollectionName(StoreOptions options) {
return options != null ? options.getCollectionNameOrDefault(collectionName) : collectionName;
}
/**
* 构建特定集合操作的URL包含tenant和database
*/
private String buildCollectionUrl(String collectionId, String operation) {
StringBuilder urlBuilder = new StringBuilder(baseUrl).append(BASE_API);
if (tenant != null && !tenant.isEmpty()) {
urlBuilder.append("/tenants/").append(tenant);
if (database != null && !database.isEmpty()) {
urlBuilder.append("/databases/").append(database);
}
}
urlBuilder.append("/collections/").append(collectionId).append("/").append(operation);
return urlBuilder.toString();
}
/**
* Close the connection to Chroma database
*/
public void close() {
// HttpClient类使用连接池管理这里可以添加额外的资源清理逻辑
logger.info("Chroma client closed");
}
/**
* 确保集合存在,如果不存在则创建
*/
private void ensureCollectionExists() throws IOException {
try {
// 尝试获取默认集合ID如果能获取到则说明集合存在
getCollectionId(collectionName);
logger.debug("Collection '{}' exists", collectionName);
} catch (IOException e) {
// 如果获取集合ID失败说明集合不存在需要创建
logger.info("Collection '{}' does not exist, creating...", collectionName);
createCollection();
logger.info("Collection '{}' created successfully", collectionName);
}
}
/**
* 构建集合列表URL包含tenant和database
*/
private String buildCollectionsUrl() {
StringBuilder urlBuilder = new StringBuilder(baseUrl).append(BASE_API);
if (tenant != null && !tenant.isEmpty()) {
urlBuilder.append("/tenants/").append(tenant);
if (database != null && !database.isEmpty()) {
urlBuilder.append("/databases/").append(database);
}
}
urlBuilder.append("/collections");
return urlBuilder.toString();
}
/**
* 函数式接口用于封装HTTP操作以支持重试
*/
private interface HttpOperation<T> {
T execute() throws IOException;
}
}

View File

@@ -0,0 +1,203 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.chroma;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
/**
* ChromaVectorStoreConfig class provides configuration for ChromaVectorStore.
*/
public class ChromaVectorStoreConfig implements DocumentStoreConfig {
private static final Logger logger = LoggerFactory.getLogger(ChromaVectorStoreConfig.class);
private String host = "localhost";
private int port = 8000;
private String collectionName;
private boolean autoCreateCollection = true;
private String apiKey;
private String tenant;
private String database;
public ChromaVectorStoreConfig() {
}
/**
* Get the host of Chroma database
*
* @return the host of Chroma database
*/
public String getHost() {
return host;
}
/**
* Set the host of Chroma database
*
* @param host the host of Chroma database
*/
public void setHost(String host) {
this.host = host;
}
/**
* Get the port of Chroma database
*
* @return the port of Chroma database
*/
public int getPort() {
return port;
}
/**
* Set the port of Chroma database
*
* @param port the port of Chroma database
*/
public void setPort(int port) {
this.port = port;
}
/**
* Get the collection name of Chroma database
*
* @return the collection name of Chroma database
*/
public String getCollectionName() {
return collectionName;
}
/**
* Set the collection name of Chroma database
*
* @param collectionName the collection name of Chroma database
*/
public void setCollectionName(String collectionName) {
this.collectionName = collectionName;
}
/**
* Get whether to automatically create the collection if it doesn't exist
*
* @return true if the collection should be created automatically, false otherwise
*/
public boolean isAutoCreateCollection() {
return autoCreateCollection;
}
/**
* Set whether to automatically create the collection if it doesn't exist
*
* @param autoCreateCollection true if the collection should be created automatically, false otherwise
*/
public void setAutoCreateCollection(boolean autoCreateCollection) {
this.autoCreateCollection = autoCreateCollection;
}
/**
* Get the API key of Chroma database
*
* @return the API key of Chroma database
*/
public String getApiKey() {
return apiKey;
}
/**
* Set the API key of Chroma database
*
* @param apiKey the API key of Chroma database
*/
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
/**
* Get the tenant of Chroma database
*
* @return the tenant of Chroma database
*/
public String getTenant() {
return tenant;
}
/**
* Set the tenant of Chroma database
*
* @param tenant the tenant of Chroma database
*/
public void setTenant(String tenant) {
this.tenant = tenant;
}
/**
* Get the database of Chroma database
*
* @return the database of Chroma database
*/
public String getDatabase() {
return database;
}
/**
* Set the database of Chroma database
*
* @param database the database of Chroma database
*/
public void setDatabase(String database) {
this.database = database;
}
@Override
public boolean checkAvailable() {
try {
URL url = new URL(getBaseUrl() + "/api/v2/heartbeat");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.setConnectTimeout(5000);
connection.setReadTimeout(5000);
if (apiKey != null && !apiKey.isEmpty()) {
connection.setRequestProperty("X-Chroma-Token", apiKey);
}
int responseCode = connection.getResponseCode();
connection.disconnect();
return responseCode == 200;
} catch (IOException e) {
logger.warn("Chroma database is not available: {}", e.getMessage());
return false;
}
}
/**
* Get the base URL of Chroma database
*
* @return the base URL of Chroma database
*/
public String getBaseUrl() {
return "http://" + host + ":" + port;
}
}

View File

@@ -0,0 +1,383 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.chroma;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.model.client.HttpClient;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
/**
* ChromaVectorStore的测试类测试文档的存储、搜索、更新和删除功能
* 包含连接检查和错误处理机制支持在无真实Chroma服务器时跳过测试
*/
public class ChromaVectorStoreTest {
private static ChromaVectorStore store;
private static String testTenant = "default_tenant";
private static String testDatabase = "default_database";
private static String testCollectionName = "test_collection";
private static boolean isChromaAvailable = false;
private static boolean useMock = false; // 设置为true可以在没有真实Chroma服务器时使用模拟模式
/**
* 在测试开始前初始化ChromaVectorStore实例
*/
@BeforeClass
public static void setUp() {
// 创建配置对象
ChromaVectorStoreConfig config = new ChromaVectorStoreConfig();
config.setHost("localhost");
config.setPort(8000);
config.setCollectionName(testCollectionName);
config.setTenant(testTenant);
config.setDatabase(testDatabase);
config.setAutoCreateCollection(true);
// 初始化存储实例
try {
store = new ChromaVectorStore(config);
System.out.println("ChromaVectorStore initialized successfully.");
// 检查连接是否可用
isChromaAvailable = checkChromaConnection(config);
if (!isChromaAvailable && !useMock) {
System.out.println("Chroma server is not available. Tests will be skipped unless useMock is set to true.");
}
} catch (Exception e) {
System.err.println("Failed to initialize ChromaVectorStore: " + e.getMessage());
e.printStackTrace();
}
}
/**
* 检查Chroma服务器连接是否可用
*/
private static boolean checkChromaConnection(ChromaVectorStoreConfig config) {
try {
String baseUrl = "http://" + config.getHost() + ":" + config.getPort();
String healthCheckUrl = baseUrl + "/api/v2/heartbeat";
HttpClient httpClient = new HttpClient();
System.out.println("Checking Chroma server connection at: " + healthCheckUrl);
// 使用较短的超时时间进行健康检查
String response = httpClient.get(healthCheckUrl);
if (response != null) {
System.out.println("Chroma server connection successful! Response: " + response);
return true;
} else {
System.out.println("Chroma server connection failed: Empty response");
return false;
}
} catch (Exception e) {
System.out.println("Chroma server connection failed: " + e.getMessage());
System.out.println("Please ensure Chroma server is running on http://" + config.getHost() + ":" + config.getPort());
System.out.println("To run tests without a real Chroma server, set 'useMock = true'");
return false;
}
}
/**
* 检查是否应该运行测试
*/
private void assumeChromaAvailable() {
Assume.assumeTrue("Chroma server is not available and mock mode is disabled",
isChromaAvailable || useMock);
}
/**
* 在所有测试完成后清理资源
*/
@AfterClass
public static void tearDown() {
if (store != null) {
try {
store.close();
System.out.println("ChromaVectorStore closed successfully.");
} catch (Exception e) {
System.err.println("Error closing ChromaVectorStore: " + e.getMessage());
}
}
}
/**
* 测试存储文档功能
*/
@Test
public void testStoreDocuments() {
assumeChromaAvailable();
System.out.println("Starting testStoreDocuments...");
// 创建测试文档
List<Document> documents = createTestDocuments();
// 如果使用模拟模式,直接返回成功结果
if (useMock) {
System.out.println("Running in mock mode. Simulating store operation.");
StoreResult mockResult = StoreResult.successWithIds(documents);
assertTrue("Mock store operation should be successful", mockResult.isSuccess());
assertEquals("All document IDs should be returned in mock mode",
documents.size(), mockResult.ids().size());
System.out.println("testStoreDocuments completed successfully in mock mode.");
return;
}
// 存储文档
try {
StoreResult result = store.doStore(documents, StoreOptions.DEFAULT);
System.out.println("Store result: " + result);
// 验证存储是否成功
assertTrue("Store operation should be successful", result.isSuccess());
assertEquals("All document IDs should be returned", documents.size(), result.ids().size());
System.out.println("testStoreDocuments completed successfully.");
} catch (Exception e) {
System.err.println("Failed to store documents: " + e.getMessage());
e.printStackTrace();
fail("Store operation failed with exception: " + e.getMessage());
}
}
/**
* 测试搜索文档功能
*/
@Test
public void testSearchDocuments() {
assumeChromaAvailable();
System.out.println("Starting testSearchDocuments...");
// 创建测试文档
List<Document> documents = createTestDocuments();
// 如果使用模拟模式
if (useMock) {
System.out.println("Running in mock mode. Simulating search operation.");
// 模拟搜索结果返回前3个文档
List<Document> mockResults = new ArrayList<>(documents.subList(0, Math.min(3, documents.size())));
for (int i = 0; i < mockResults.size(); i++) {
mockResults.get(i).setScore(1.0 - i * 0.1); // 模拟相似度分数
}
// 验证模拟结果
assertNotNull("Mock search results should not be null", mockResults);
assertFalse("Mock search results should not be empty", mockResults.isEmpty());
assertTrue("Mock search results should have the correct maximum size", mockResults.size() <= 3);
System.out.println("testSearchDocuments completed successfully in mock mode.");
return;
}
try {
// 首先存储一些测试文档
store.doStore(documents, StoreOptions.DEFAULT);
// 创建搜索包装器
SearchWrapper searchWrapper = new SearchWrapper();
// 使用第一个文档的向量进行搜索
searchWrapper.setVector(documents.get(0).getVector());
searchWrapper.setMaxResults(3);
// 执行搜索
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
// 验证搜索结果
assertNotNull("Search results should not be null", searchResults);
assertFalse("Search results should not be empty", searchResults.isEmpty());
assertTrue("Search results should have the correct maximum size",
searchResults.size() <= searchWrapper.getMaxResults());
// 打印搜索结果
System.out.println("Search results:");
for (Document doc : searchResults) {
System.out.printf("id=%s, content=%s, vector=%s, score=%s\n",
doc.getId(), doc.getContent(), Arrays.toString(doc.getVector()), doc.getScore());
}
System.out.println("testSearchDocuments completed successfully.");
} catch (Exception e) {
System.err.println("Failed to search documents: " + e.getMessage());
e.printStackTrace();
fail("Search operation failed with exception: " + e.getMessage());
}
}
/**
* 测试更新文档功能
*/
@Test
public void testUpdateDocuments() {
assumeChromaAvailable();
System.out.println("Starting testUpdateDocuments...");
// 创建测试文档
List<Document> documents = createTestDocuments();
// 如果使用模拟模式
if (useMock) {
System.out.println("Running in mock mode. Simulating update operation.");
// 修改文档内容
Document updatedDoc = documents.get(0);
String originalContent = updatedDoc.getContent();
updatedDoc.setContent(originalContent + " [UPDATED]");
// 模拟更新结果
StoreResult mockResult = StoreResult.successWithIds(Arrays.asList(updatedDoc));
assertTrue("Mock update operation should be successful", mockResult.isSuccess());
System.out.println("testUpdateDocuments completed successfully in mock mode.");
return;
}
try {
// 首先存储一些测试文档
store.doStore(documents, StoreOptions.DEFAULT);
// 修改文档内容
Document updatedDoc = documents.get(0);
String originalContent = updatedDoc.getContent();
updatedDoc.setContent(originalContent + " [UPDATED]");
// 执行更新
StoreResult result = store.doUpdate(Arrays.asList(updatedDoc), StoreOptions.DEFAULT);
// 验证更新是否成功
assertTrue("Update operation should be successful", result.isSuccess());
// 搜索更新后的文档以验证更改
SearchWrapper searchWrapper = new SearchWrapper();
searchWrapper.setVector(updatedDoc.getVector());
searchWrapper.setMaxResults(1);
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
assertTrue("Should find the updated document", !searchResults.isEmpty());
assertEquals("Document content should be updated",
updatedDoc.getContent(), searchResults.get(0).getContent());
System.out.println("testUpdateDocuments completed successfully.");
} catch (Exception e) {
System.err.println("Failed to update documents: " + e.getMessage());
e.printStackTrace();
fail("Update operation failed with exception: " + e.getMessage());
}
}
/**
* 测试删除文档功能
*/
@Test
public void testDeleteDocuments() {
assumeChromaAvailable();
System.out.println("Starting testDeleteDocuments...");
// 创建测试文档
List<Document> documents = createTestDocuments();
// 如果使用模拟模式
if (useMock) {
System.out.println("Running in mock mode. Simulating delete operation.");
// 获取要删除的文档ID
List<Object> idsToDelete = new ArrayList<>();
idsToDelete.add(documents.get(0).getId());
// 模拟删除结果
StoreResult mockResult = StoreResult.success();
assertTrue("Mock delete operation should be successful", mockResult.isSuccess());
System.out.println("testDeleteDocuments completed successfully in mock mode.");
return;
}
try {
// 首先存储一些测试文档
store.doStore(documents, StoreOptions.DEFAULT);
// 获取要删除的文档ID
List<Object> idsToDelete = new ArrayList<>();
idsToDelete.add(documents.get(0).getId());
// 执行删除
StoreResult result = store.doDelete(idsToDelete, StoreOptions.DEFAULT);
// 验证删除是否成功
assertTrue("Delete operation should be successful", result.isSuccess());
// 尝试搜索已删除的文档
SearchWrapper searchWrapper = new SearchWrapper();
searchWrapper.setVector(documents.get(0).getVector());
searchWrapper.setMaxResults(10);
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
// 检查结果中是否包含已删除的文档
boolean deletedDocFound = searchResults.stream()
.anyMatch(doc -> doc.getId().equals(documents.get(0).getId()));
assertFalse("Deleted document should not be found", deletedDocFound);
System.out.println("testDeleteDocuments completed successfully.");
} catch (Exception e) {
System.err.println("Failed to delete documents: " + e.getMessage());
e.printStackTrace();
fail("Delete operation failed with exception: " + e.getMessage());
}
}
/**
* 创建测试文档
*/
private List<Document> createTestDocuments() {
List<Document> documents = new ArrayList<>();
// 创建5个测试文档每个文档都有不同的内容和向量
for (int i = 0; i < 5; i++) {
Document doc = new Document();
doc.setId("doc_" + i);
doc.setContent("This is test document content " + i);
doc.setTitle("Test Document " + i);
// 创建一个简单的向量向量维度为10
float[] vector = new float[10];
for (int j = 0; j < vector.length; j++) {
vector[j] = i + j * 0.1f;
}
doc.setVector(vector);
documents.add(doc);
}
return documents;
}
}

View File

@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-elasticsearch</name>
<artifactId>easy-agents-store-elasticsearch</artifactId>
<properties>
<elasticsearch.version>8.15.0</elasticsearch.version>
<jackson.version>2.17.0</jackson.version>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,282 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.elasticsearch;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ErrorCause;
import co.elastic.clients.elasticsearch._types.mapping.DenseVectorProperty;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch._types.mapping.TextProperty;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.ScriptScoreQuery;
import co.elastic.clients.elasticsearch.core.BulkRequest;
import co.elastic.clients.elasticsearch.core.BulkResponse;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem;
import co.elastic.clients.json.JsonData;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.endpoints.BooleanResponse;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.store.exception.StoreException;
import com.easyagents.core.util.StringUtil;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.message.BasicHeader;
import org.apache.http.ssl.SSLContextBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* es 向量存储:<a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/introduction.html">elasticsearch-java</a>
*
* @author songyinyin
* @since 2024/8/12 下午4:17
*/
public class ElasticSearchVectorStore extends DocumentStore {
private static final Logger log = LoggerFactory.getLogger(ElasticSearchVectorStore.class);
private final ElasticsearchClient client;
private final ElasticSearchVectorStoreConfig config;
public ElasticSearchVectorStore(ElasticSearchVectorStoreConfig config) {
this.config = config;
RestClientBuilder restClientBuilder = RestClient.builder(HttpHost.create(config.getServerUrl()));
try {
SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build();
if (StringUtil.hasText(config.getUsername())) {
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword()));
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setSSLContext(sslContext);
httpClientBuilder.setDefaultCredentialsProvider(provider);
return httpClientBuilder;
});
}
if (StringUtil.hasText(config.getApiKey())) {
restClientBuilder.setDefaultHeaders(new Header[]{
new BasicHeader("Authorization", "Apikey " + config.getApiKey())
});
}
ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper());
this.client = new ElasticsearchClient(transport);
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
throw new StoreException("Elasticsearch init error", e);
}
try {
client.ping();
} catch (IOException e) {
log.error("[I/O Elasticsearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
public ElasticSearchVectorStore(ElasticSearchVectorStoreConfig config, ElasticsearchClient client) {
this.config = config;
this.client = client;
}
private static void throwIfError(BulkResponse bulkResponse) {
if (bulkResponse.errors()) {
for (BulkResponseItem item : bulkResponse.items()) {
if (item.error() == null) {
continue;
}
ErrorCause errorCause = item.error();
throw new StoreException("type: " + errorCause.type() + "," + "reason: " + errorCause.reason());
}
}
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
String indexName;
if (StringUtil.hasText(options.getCollectionName())){
indexName = options.getCollectionName();
} else {
indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
}
createIndexIfNotExist(indexName);
return saveOrUpdate(documents, indexName);
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
for (Object id : ids) {
bulkBuilder.operations(op -> op.delete(d -> d.index(indexName).id(id.toString())));
}
bulk(bulkBuilder.build());
return StoreResult.success();
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
return saveOrUpdate(documents, indexName);
}
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
// 最小匹配分数无值则默认0
Double minScore = wrapper.getMinScore();
// 获取索引名,无指定则使用配置的默认索引
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
// 公式:(cosineSimilarity + 1.0) / 2 将相似度映射到 0~1 区间
ScriptScoreQuery scriptScoreQuery = ScriptScoreQuery.of(fn -> fn
.minScore(minScore == null ? 0 : minScore.floatValue())
.query(Query.of(q -> q.matchAll(m -> m)))
.script(s -> s
.source("(cosineSimilarity(params.query_vector, 'vector') + 1.0) / 2")
.params("query_vector", JsonData.of(wrapper.getVector()))
)
);
try {
SearchResponse<JsonData> response = client.search(
SearchRequest.of(s -> s.index(indexName)
.query(n -> n.scriptScore(scriptScoreQuery))
.size(wrapper.getMaxResults())),
JsonData.class
);
return response.hits().hits().stream()
.filter(hit -> hit.source() != null) // 过滤_source为空的无效结果
.map(hit -> parseFromJsonData(hit.source(), hit.score()))
.collect(Collectors.toList());
} catch (IOException e) {
log.error("[es/search] Elasticsearch I/O exception occurred", e);
throw new StoreException(e.getMessage());
}
}
private StoreResult saveOrUpdate(List<Document> documents, String indexName) {
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
for (Document document : documents) {
bulkBuilder.operations(op -> op.index(
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
);
}
bulk(bulkBuilder.build());
return StoreResult.successWithIds(documents);
}
private void bulk(BulkRequest bulkRequest) {
try {
BulkResponse bulkResponse = client.bulk(bulkRequest);
throwIfError(bulkResponse);
} catch (IOException e) {
log.error("[I/O Elasticsearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
private void createIndexIfNotExist(String indexName) {
try {
BooleanResponse response = client.indices().exists(c -> c.index(indexName));
if (!response.value()) {
log.info("[ElasticSearch] Index {} not exists, creating...", indexName);
client.indices().create(c -> c.index(indexName)
.mappings(getDefaultMappings(this.getEmbeddingModel().dimensions())));
}
} catch (IOException e) {
log.error("[I/O ElasticSearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
private TypeMapping getDefaultMappings(int dimension) {
Map<String, Property> properties = new HashMap<>(4);
properties.put("content", Property.of(p -> p.text(TextProperty.of(t -> t))));
properties.put("vector", Property.of(p -> p.denseVector(DenseVectorProperty.of(d -> d.dims(dimension)))));
return TypeMapping.of(c -> c.properties(properties));
}
private Document parseFromJsonData(JsonData source, Double score) {
Document document = new Document();
Map<String, Object> dataMap = source.to(Map.class);
document.setId(dataMap.get("id"));
document.setTitle((String) dataMap.get("title"));
document.setContent((String) dataMap.get("content"));
document.setScore(score);
Object vectorObj = dataMap.get("vector");
if (vectorObj instanceof List<?>) {
List<?> vectorList = (List<?>) vectorObj;
float[] vector = new float[vectorList.size()];
for (int i = 0; i < vectorList.size(); i++) {
Object val = vectorList.get(i);
if (val instanceof Number) {
vector[i] = ((Number) val).floatValue();
}
}
document.setVector(vector);
}
@SuppressWarnings("unchecked")
Map<String, Object> metadataMap = (Map<String, Object>) dataMap.get("metadataMap");
if (metadataMap != null && !metadataMap.isEmpty()) {
document.setMetadataMap(metadataMap);
} else {
Map<String, Object> otherMetadata = new HashMap<>();
for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
String key = entry.getKey();
if (!"id".equals(key) && !"title".equals(key)
&& !"content".equals(key) && !"vector".equals(key)) {
otherMetadata.put(key, entry.getValue());
}
}
if (!otherMetadata.isEmpty()) {
document.setMetadataMap(otherMetadata);
}
}
return document;
}
}

View File

@@ -0,0 +1,82 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.elasticsearch;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
/**
* 连接 elasticsearch 配置:<a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/getting-started-java.html">elasticsearch-java</a>
*
* @author songyinyin
*/
public class ElasticSearchVectorStoreConfig implements DocumentStoreConfig {
private String serverUrl = "https://localhost:9200";
private String apiKey;
private String username;
private String password;
private String defaultIndexName = "easy-agents-default";
public String getServerUrl() {
return serverUrl;
}
public void setServerUrl(String serverUrl) {
this.serverUrl = serverUrl;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getDefaultIndexName() {
return defaultIndexName;
}
public void setDefaultIndexName(String defaultIndexName) {
this.defaultIndexName = defaultIndexName;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.serverUrl, this.apiKey, this.defaultIndexName);
}
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.opensearch;
import com.easyagents.core.document.Document;
import com.easyagents.core.model.embedding.EmbeddingModel;
import com.easyagents.core.model.embedding.EmbeddingOptions;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.VectorData;
import com.easyagents.core.store.exception.StoreException;
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author songyinyin
*/
public class ElasticSearchVectorStoreTest {
private static ElasticSearchVectorStore getVectorStore() {
ElasticSearchVectorStoreConfig config = new ElasticSearchVectorStoreConfig();
// config.setApiKey("bmtXRVNaRUJNMEZXZzMzcnNvSXk6MlNMVmFnT0hRVVNUSmN3UXpoNWp4Zw==");
config.setUsername("elastic");
config.setPassword("Dd2024a10");
ElasticSearchVectorStore store = new ElasticSearchVectorStore(config);
store.setEmbeddingModel(new EmbeddingModel() {
@Override
public VectorData embed(Document document, EmbeddingOptions options) {
VectorData vectorData = new VectorData();
vectorData.setVector(new float[]{0, 0});
return vectorData;
}
});
return store;
}
@Test(expected = StoreException.class)
public void test01() {
ElasticSearchVectorStore store = getVectorStore();
// https://opensearch.org/docs/latest/search-plugins/vector-search/#example
List<Document> list = new ArrayList<>();
Document doc1 = new Document();
doc1.setId(1);
doc1.setContent("test1");
doc1.setVector(new float[]{5.2f, 4.4f});
list.add(doc1);
Document doc2 = new Document();
doc2.setId(2);
doc2.setContent("test2");
doc2.setVector(new float[]{5.2f, 3.9f});
list.add(doc2);
Document doc3 = new Document();
doc3.setId(3);
doc3.setContent("test3");
doc3.setVector(new float[]{4.9f, 3.4f});
list.add(doc3);
Document doc4 = new Document();
doc4.setId(4);
doc4.setContent("test4");
doc4.setVector(new float[]{4.2f, 4.6f});
list.add(doc4);
Document doc5 = new Document();
doc5.setId(5);
doc5.setContent("test5");
doc5.setVector(new float[]{3.3f, 4.5f});
list.add(doc5);
store.doStore(list, StoreOptions.DEFAULT);
// 可能要等一会 才能查出结果
SearchWrapper searchWrapper = new SearchWrapper();
searchWrapper.setVector(new float[]{5, 4});
searchWrapper.setMaxResults(3);
List<Document> documents = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
for (Document document : documents) {
System.out.printf("id=%s, content=%s, vector=%s, metadata=%s\n",
document.getId(), document.getContent(), Arrays.toString(document.getVector()), document.getMetadataMap());
}
}
}

View File

@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-opensearch</name>
<artifactId>easy-agents-store-opensearch</artifactId>
<properties>
<opensearch.version>2.13.0</opensearch.version>
<httpclient5.version>5.1.4</httpclient5.version>
<httpcore5-h2>5.1.4</httpcore5-h2>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-java</artifactId>
<version>${opensearch.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>${httpclient5.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.core5</groupId>
<artifactId>httpcore5-h2</artifactId>
<version>${httpcore5-h2}</version> <!-- 版本需与 httpclient5 一致 -->
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,264 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.opensearch;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.store.exception.StoreException;
import com.easyagents.core.util.StringUtil;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
import org.apache.hc.core5.ssl.SSLContextBuilder;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch._types.ErrorCause;
import org.opensearch.client.opensearch._types.InlineScript;
import org.opensearch.client.opensearch._types.mapping.Property;
import org.opensearch.client.opensearch._types.mapping.TextProperty;
import org.opensearch.client.opensearch._types.mapping.TypeMapping;
import org.opensearch.client.opensearch._types.query_dsl.Query;
import org.opensearch.client.opensearch._types.query_dsl.ScriptScoreQuery;
import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.BulkResponse;
import org.opensearch.client.opensearch.core.SearchRequest;
import org.opensearch.client.opensearch.core.SearchResponse;
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
import org.opensearch.client.transport.OpenSearchTransport;
import org.opensearch.client.transport.endpoints.BooleanResponse;
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
/**
* OpenSearch 向量存储
*
* @author songyinyin
* @since 2024/8/10 下午8:31
*/
public class OpenSearchVectorStore extends DocumentStore {
private static final Logger log = LoggerFactory.getLogger(OpenSearchVectorStore.class);
private final OpenSearchClient client;
private final OpenSearchVectorStoreConfig config;
public OpenSearchVectorStore(OpenSearchVectorStoreConfig config) {
this.config = config;
HttpHost openSearchHost;
try {
openSearchHost = HttpHost.create(config.getServerUrl());
} catch (URISyntaxException se) {
log.error("[OpenSearch Exception]", se);
throw new StoreException(se.getMessage());
}
try {
SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build();
TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
.setSslContext(sslContext)
.setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
.build();
OpenSearchTransport transport = ApacheHttpClient5TransportBuilder
.builder(openSearchHost)
.setMapper(new JacksonJsonpMapper())
.setHttpClientConfigCallback(httpClientBuilder -> {
if (StringUtil.hasText(config.getApiKey())) {
httpClientBuilder.setDefaultHeaders(singletonList(
new BasicHeader("Authorization", "ApiKey " + config.getApiKey())
));
}
if (StringUtil.hasText(config.getUsername()) && StringUtil.hasText(config.getPassword())) {
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(openSearchHost),
new UsernamePasswordCredentials(config.getUsername(), config.getPassword().toCharArray()));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
httpClientBuilder.setConnectionManager(PoolingAsyncClientConnectionManagerBuilder
.create().setTlsStrategy(tlsStrategy).build());
return httpClientBuilder;
})
.build();
this.client = new OpenSearchClient(transport);
try {
client.ping();
} catch (IOException e) {
log.error("[I/O OpenSearch Exception]", e);
throw new StoreException(e.getMessage());
}
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
throw new StoreException("OpenSearchClient init error", e);
}
}
public OpenSearchVectorStore(OpenSearchVectorStoreConfig config, OpenSearchClient client) {
this.config = config;
this.client = client;
}
private void createIndexIfNotExist(String indexName) {
try {
BooleanResponse response = client.indices().exists(c -> c.index(indexName));
if (!response.value()) {
log.info("[OpenSearch] Index {} not exists, creating...", indexName);
client.indices().create(c -> c.index(indexName)
.settings(s -> s.knn(true))
.mappings(getDefaultMappings(this.getEmbeddingModel().dimensions())));
}
} catch (IOException e) {
log.error("[I/O OpenSearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
private TypeMapping getDefaultMappings(int dimension) {
Map<String, Property> properties = new HashMap<>(4);
properties.put("content", Property.of(p -> p.text(TextProperty.of(t -> t))));
properties.put("vector", Property.of(p -> p.knnVector(
k -> k.dimension(dimension)
)));
return TypeMapping.of(c -> c.properties(properties));
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
createIndexIfNotExist(indexName);
for (Document document : documents) {
bulkBuilder.operations(op -> op.index(
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
);
}
bulk(bulkBuilder.build());
return StoreResult.successWithIds(documents);
}
private void bulk(BulkRequest bulkRequest) {
try {
BulkResponse bulkResponse = client.bulk(bulkRequest);
throwIfError(bulkResponse);
} catch (IOException e) {
log.error("[I/O OpenSearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
private static void throwIfError(BulkResponse bulkResponse) {
if (bulkResponse.errors()) {
for (BulkResponseItem item : bulkResponse.items()) {
if (item.error() == null) {
continue;
}
ErrorCause errorCause = item.error();
throw new StoreException("type: " + errorCause.type() + "," + "reason: " + errorCause.reason());
}
}
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
for (Object id : ids) {
bulkBuilder.operations(op -> op.delete(d -> d.index(indexName).id(id.toString())));
}
bulk(bulkBuilder.build());
return StoreResult.success();
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
for (Document document : documents) {
bulkBuilder.operations(op -> op.update(
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
);
}
bulk(bulkBuilder.build());
return StoreResult.successWithIds(documents);
}
@Override
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
Double minScore = wrapper.getMinScore();
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
// https://aws.amazon.com/cn/blogs/china/use-aws-opensearch-knn-plug-in-to-implement-vector-retrieval/
// boost 默认是 1小于 1 会降低相关性: https://opensearch.org/docs/latest/query-dsl/specialized/script-score/#parameters
ScriptScoreQuery scriptScoreQuery = ScriptScoreQuery.of(q -> q.minScore(minScore == null ? 0 : minScore.floatValue())
.query(Query.of(qu -> qu.matchAll(m -> m)))
.script(s -> s.inline(InlineScript.of(i -> i
.source("knn_score")
.lang("knn")
.params("field", JsonData.of("vector"))
.params("query_value", JsonData.of(wrapper.getVector()))
.params("space_type", JsonData.of("cosinesimil"))
))));
try {
SearchResponse<Document> response = client.search(
SearchRequest.of(s -> s.index(indexName)
.query(n -> n.scriptScore(scriptScoreQuery))
.size(wrapper.getMaxResults())),
Document.class
);
return response.hits().hits().stream()
.filter(s -> s.source() != null)
.map(s -> {
Document source = s.source();
source.setScore(s.score());
return source;
})
.collect(toList());
} catch (IOException e) {
log.error("[I/O OpenSearch Exception]", e);
throw new StoreException(e.getMessage());
}
}
}

View File

@@ -0,0 +1,85 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.opensearch;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
/**
* 连接 open search 配置:<a href="https://opensearch.org/docs/latest/clients/java/">opensearch-java</a>
*
* @author songyinyin
* @since 2024/8/10 下午8:39
*/
public class OpenSearchVectorStoreConfig implements DocumentStoreConfig {
private String serverUrl = "https://localhost:9200";
private String apiKey;
private String username;
private String password;
private String defaultIndexName = "easy-agents-default";
public String getServerUrl() {
return serverUrl;
}
public void setServerUrl(String serverUrl) {
this.serverUrl = serverUrl;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getDefaultIndexName() {
return defaultIndexName;
}
public void setDefaultIndexName(String defaultIndexName) {
this.defaultIndexName = defaultIndexName;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.serverUrl, this.apiKey)
|| StringUtil.hasText(this.serverUrl, this.username, this.password);
}
}

View File

@@ -0,0 +1,97 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.opensearch;
import com.easyagents.core.document.Document;
import com.easyagents.core.model.embedding.EmbeddingModel;
import com.easyagents.core.model.embedding.EmbeddingOptions;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.VectorData;
import com.easyagents.core.store.exception.StoreException;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author songyinyin
* @since 2024/8/11 下午3:21
*/
public class OpenSearchVectorStoreTest {
@Test(expected = StoreException.class)
public void test01() {
OpenSearchVectorStore store = getOpenSearchVectorStore();
// https://opensearch.org/docs/latest/search-plugins/vector-search/#example
List<Document> list = new ArrayList<>();
Document doc1 = new Document();
doc1.setId(1);
doc1.setContent("test1");
doc1.setVector(new float[]{5.2f, 4.4f});
list.add(doc1);
Document doc2 = new Document();
doc2.setId(2);
doc2.setContent("test2");
doc2.setVector(new float[]{5.2f, 3.9f});
list.add(doc2);
Document doc3 = new Document();
doc3.setId(3);
doc3.setContent("test3");
doc3.setVector(new float[]{4.9f, 3.4f});
list.add(doc3);
Document doc4 = new Document();
doc4.setId(4);
doc4.setContent("test4");
doc4.setVector(new float[]{4.2f, 4.6f});
list.add(doc4);
Document doc5 = new Document();
doc5.setId(5);
doc5.setContent("test5");
doc5.setVector(new float[]{3.3f, 4.5f});
list.add(doc5);
store.doStore(list, StoreOptions.DEFAULT);
// 可能要等一会 才能查出结果
SearchWrapper searchWrapper = new SearchWrapper();
searchWrapper.setVector(new float[]{5, 4});
searchWrapper.setMaxResults(3);
List<Document> documents = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
for (Document document : documents) {
System.out.printf("id=%s, content=%s, vector=%s, metadata=%s\n",
document.getId(), document.getContent(), Arrays.toString(document.getVector()), document.getMetadataMap());
}
}
private static OpenSearchVectorStore getOpenSearchVectorStore() {
OpenSearchVectorStoreConfig config = new OpenSearchVectorStoreConfig();
config.setUsername("admin");
config.setPassword("4_Pa46WQczS?");
OpenSearchVectorStore store = new OpenSearchVectorStore(config);
store.setEmbeddingModel(new EmbeddingModel() {
@Override
public VectorData embed(Document document, EmbeddingOptions options) {
VectorData vectorData = new VectorData();
vectorData.setVector(new float[]{0, 0});
return vectorData;
}
});
return store;
}
}

View File

@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-pgvector</name>
<artifactId>easy-agents-store-pgvector</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,63 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.pgvector;
import org.postgresql.util.PGobject;
import java.sql.SQLException;
public class PgvectorUtil {
/**
* 转化为vector.
* 如果需要half vector或者sparse vector 对应实现即可
* @param src 向量
* @return
* @throws SQLException
*/
public static PGobject toPgVector(double[] src) throws SQLException {
PGobject vector = new PGobject();
vector.setType("vector");
if (src.length == 0) {
vector.setValue("[]");
return vector;
}
StringBuilder sb = new StringBuilder("[");
for (double v : src) {
sb.append(v);
sb.append(",");
}
vector.setValue(sb.substring(0, sb.length() - 1) + "]");
return vector;
}
public static double[] fromPgVector(String src) {
if (src.equals("[]")) {
return new double[0];
}
String[] strs = src.substring(1, src.length() - 1).split(",");
double[] output = new double[strs.length];
for (int i = 0; i < strs.length; i++) {
try {
output[i] = Double.parseDouble(strs[i]);
} catch (Exception ignore) {
output[i] = 0;
}
}
return output;
}
}

View File

@@ -0,0 +1,231 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.pgvector;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import org.postgresql.ds.PGSimpleDataSource;
import org.postgresql.util.PGobject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.*;
public class PgvectorVectorStore extends DocumentStore {
private static final Logger logger = LoggerFactory.getLogger(PgvectorVectorStore.class);
public static final double DEFAULT_SIMILARITY_THRESHOLD = 0.3;
private final PGSimpleDataSource dataSource;
private final String defaultCollectionName;
private final PgvectorVectorStoreConfig config;
public PgvectorVectorStore(PgvectorVectorStoreConfig config) {
dataSource = new PGSimpleDataSource();
dataSource.setServerNames(new String[]{config.getHost() + ":" + config.getPort()});
dataSource.setUser(config.getUsername());
dataSource.setPassword(config.getPassword());
dataSource.setDatabaseName(config.getDatabaseName());
if (!config.getProperties().isEmpty()) {
config.getProperties().forEach((k, v) -> {
try {
dataSource.setProperty(k, v);
} catch (SQLException e) {
logger.error("set pg property error", e);
}
});
}
this.defaultCollectionName = config.getDefaultCollectionName();
this.config = config;
// 异步初始化数据库
new Thread(this::initDb).start();
}
public void initDb() {
// 启动的时候初始化向量表, 需要数据库支持pgvector插件
// pg管理员需要在对应的库上执行 CREATE EXTENSION IF NOT EXISTS vector;
if (config.isAutoCreateCollection()) {
createCollection(defaultCollectionName);
}
}
private Connection getConnection() throws SQLException {
Connection connection = dataSource.getConnection();
connection.setAutoCommit(false);
return connection;
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
// 表名
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
try (Connection connection = getConnection()) {
PreparedStatement pstmt = connection.prepareStatement("insert into " + collectionName + " (id, content, vector, metadata) values (?, ?, ?, ?::jsonb)");
for (Document doc : documents) {
Map<String, Object> metadatas = doc.getMetadataMap();
JSONObject jsonObject = JSON.parseObject(JSON.toJSONBytes(metadatas == null ? Collections.EMPTY_MAP : metadatas));
pstmt.setString(1, String.valueOf(doc.getId()));
pstmt.setString(2, doc.getContent());
pstmt.setObject(3, PgvectorUtil.toPgVector(doc.getVectorAsDoubleArray()));
pstmt.setString(4, jsonObject.toString());
pstmt.addBatch();
}
pstmt.executeBatch();
connection.commit();
} catch (SQLException e) {
logger.error("store vector error", e);
return StoreResult.fail();
}
return StoreResult.successWithIds(documents);
}
private Boolean createCollection(String collectionName) {
try (Connection connection = getConnection()) {
try (CallableStatement statement = connection.prepareCall("CREATE TABLE IF NOT EXISTS " + collectionName +
" (id varchar(100) PRIMARY KEY, content text, vector vector(" + config.getVectorDimension() + "), metadata jsonb)")) {
statement.execute();
}
// 默认情况下pgvector 执行精确的最近邻搜索,从而提供完美的召回率. 可以通过索引来修改 pgvector 的搜索方式,以获得更好的性能。
// By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
if (config.isUseHnswIndex()) {
try (Statement stmt = connection.createStatement()) {
stmt.execute("CREATE INDEX IF NOT EXISTS " + collectionName + "_vector_idx ON " + collectionName +
" USING hnsw (vector vector_cosine_ops)");
}
}
} catch (SQLException e) {
logger.error("create collection error", e);
return false;
}
return true;
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
StringBuilder sql = new StringBuilder("DELETE FROM " + options.getCollectionNameOrDefault(defaultCollectionName) + " WHERE id IN (");
for (int i = 0; i < ids.size(); i++) {
sql.append("?");
if (i < ids.size() - 1) {
sql.append(",");
}
}
sql.append(")");
try (Connection connection = getConnection()) {
PreparedStatement pstmt = connection.prepareStatement(sql.toString());
ArrayList<?> list = new ArrayList<>(ids);
for (int i = 0; i < list.size(); i++) {
pstmt.setString(i + 1, (String) list.get(i));
}
pstmt.executeUpdate();
connection.commit();
} catch (Exception e) {
logger.error("delete document error: " + e, e);
return StoreResult.fail();
}
return StoreResult.success();
}
@Override
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
StringBuilder sql = new StringBuilder("select ");
if (searchWrapper.isOutputVector()) {
sql.append("id, vector, content, metadata");
} else {
sql.append("id, content, metadata");
}
sql.append(" from ").append(options.getCollectionNameOrDefault(defaultCollectionName));
sql.append(" where vector <=> ? < ? order by vector <=> ? LIMIT ?");
try (Connection connection = getConnection()){
// 使用余弦距离计算最相似的文档
PreparedStatement stmt = connection.prepareStatement(sql.toString());
PGobject vector = PgvectorUtil.toPgVector(searchWrapper.getVectorAsDoubleArray());
stmt.setObject(1, vector);
stmt.setObject(2, Optional.ofNullable(searchWrapper.getMinScore()).orElse(DEFAULT_SIMILARITY_THRESHOLD));
stmt.setObject(3, vector);
stmt.setObject(4, searchWrapper.getMaxResults());
ResultSet resultSet = stmt.executeQuery();
List<Document> documents = new ArrayList<>();
while (resultSet.next()) {
Document doc = new Document();
doc.setId(resultSet.getString("id"));
doc.setContent(resultSet.getString("content"));
doc.addMetadata(JSON.parseObject(resultSet.getString("metadata")));
if (searchWrapper.isOutputVector()) {
String vectorStr = resultSet.getString("vector");
doc.setVector(PgvectorUtil.fromPgVector(vectorStr));
}
documents.add(doc);
}
return documents;
} catch (Exception e) {
logger.error("Error searching in pgvector", e);
return Collections.emptyList();
}
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
StringBuilder sql = new StringBuilder("UPDATE " + options.getCollectionNameOrDefault(defaultCollectionName) + " SET ");
sql.append("content = ?, vector = ?, metadata = ?::jsonb WHERE id = ?");
try (Connection connection = getConnection()) {
PreparedStatement pstmt = connection.prepareStatement(sql.toString());
for (Document doc : documents) {
Map<String, Object> metadatas = doc.getMetadataMap();
JSONObject metadataJson = JSON.parseObject(JSON.toJSONBytes(metadatas == null ? Collections.EMPTY_MAP : metadatas));
pstmt.setString(1, doc.getContent());
pstmt.setObject(2, PgvectorUtil.toPgVector(doc.getVectorAsDoubleArray()));
pstmt.setString(3, metadataJson.toString());
pstmt.setString(4, String.valueOf(doc.getId()));
pstmt.addBatch();
}
pstmt.executeUpdate();
connection.commit();
} catch (Exception e) {
logger.error("Error update in pgvector", e);
return StoreResult.fail();
}
return StoreResult.successWithIds(documents);
}
}

View File

@@ -0,0 +1,127 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.pgvector;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
import java.util.HashMap;
import java.util.Map;
/**
* postgreSQL访问配置
* https://github.com/pgvector/pgvector
*/
public class PgvectorVectorStoreConfig implements DocumentStoreConfig {
private String host;
private int port = 5432;
private String databaseName = "agent_vector";
private String username;
private String password;
private Map<String, String> properties = new HashMap<>();
private String defaultCollectionName;
private boolean autoCreateCollection = true;
private boolean useHnswIndex = false;
private int vectorDimension = 1024;
public PgvectorVectorStoreConfig() {
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getDatabaseName() {
return databaseName;
}
public void setDatabaseName(String databaseName) {
this.databaseName = databaseName;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
public boolean isAutoCreateCollection() {
return autoCreateCollection;
}
public void setAutoCreateCollection(boolean autoCreateCollection) {
this.autoCreateCollection = autoCreateCollection;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public Map<String, String> getProperties() {
return properties;
}
public void setProperties(Map<String, String> properties) {
this.properties = properties;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.host, this.username, this.password, this.databaseName);
}
public int getVectorDimension() {
return vectorDimension;
}
public void setVectorDimension(int vectorDimension) {
this.vectorDimension = vectorDimension;
}
public boolean isUseHnswIndex() {
return useHnswIndex;
}
public void setUseHnswIndex(boolean useHnswIndex) {
this.useHnswIndex = useHnswIndex;
}
}

View File

@@ -0,0 +1,134 @@
package com.easyagents.store.pgvector;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.Maps;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class PgvectorDbTest {
@Test
public void testInsert() {
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
config.setHost("127.0.0.1");
config.setPort(5432);
config.setDatabaseName("pgvector_test");
config.setUsername("test");
config.setPassword("123456");
config.setVectorDimension(1024);
config.setUseHnswIndex(true);
config.setAutoCreateCollection(true);
config.setDefaultCollectionName("test");
PgvectorVectorStore store = new PgvectorVectorStore(config);
Document doc = new Document("测试数据");
// 初始化 vector 为长度为 1024 的全是 1 的数组
float[] vector = new float[1024];
Arrays.fill(vector, 1.0f);
doc.setVector(vector);
doc.setMetadataMap(Maps.of("test", "test"));
store.store(doc);
}
@Test
public void testInsertMany() {
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
config.setHost("127.0.0.1");
config.setPort(5432);
config.setDatabaseName("pgvector_test");
config.setUsername("test");
config.setPassword("123456");
config.setVectorDimension(1024);
config.setUseHnswIndex(true);
config.setAutoCreateCollection(true);
config.setDefaultCollectionName("test");
PgvectorVectorStore store = new PgvectorVectorStore(config);
List<Document> docs = new ArrayList<>(100);
for (int i = 0; i < 100; i++) {
Document doc = new Document("测试数据" + i);
// 初始化 vector 为长度为 1024 的全是 1 的数组
float[] vector = new float[1024];
Arrays.fill(vector, (float) Math.random());
doc.setVector(vector);
doc.setMetadataMap(Maps.of("test", "test" + i));
docs.add(doc);
}
store.store(docs);
}
@Test
public void testSearch() {
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
config.setHost("127.0.0.1");
config.setPort(5432);
config.setDatabaseName("pgvector_test");
config.setUsername("test");
config.setPassword("123456");
config.setVectorDimension(1024);
config.setUseHnswIndex(true);
config.setAutoCreateCollection(true);
config.setDefaultCollectionName("test");
PgvectorVectorStore store = new PgvectorVectorStore(config);
float[] vector = new float[1024];
Arrays.fill(vector, 1.0f);
SearchWrapper searchWrapper = new SearchWrapper().text("测试数据");
searchWrapper.setVector(vector);
searchWrapper.setMinScore(0.0);
searchWrapper.setOutputVector(true);
List<Document> docs = store.search(searchWrapper);
System.out.println(docs);
}
@Test
public void testUpdate() {
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
config.setHost("127.0.0.1");
config.setPort(5432);
config.setDatabaseName("pgvector_test");
config.setUsername("test");
config.setPassword("123456");
config.setVectorDimension(1024);
config.setUseHnswIndex(true);
config.setAutoCreateCollection(true);
config.setDefaultCollectionName("test");
PgvectorVectorStore store = new PgvectorVectorStore(config);
Document document = new Document("测试数据");
document.setId("145314895749100ae8306079519b3393");
document.setMetadataMap(Maps.of("test", "test0"));
float[] vector = new float[1024];
Arrays.fill(vector, 1.1f);
document.setVector(vector);
StoreResult update = store.update(document);
System.out.println(update);
}
@Test
public void testDelete() {
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
config.setHost("127.0.0.1");
config.setPort(5432);
config.setDatabaseName("pgvector_test");
config.setUsername("test");
config.setPassword("123456");
config.setVectorDimension(1024);
config.setUseHnswIndex(true);
config.setAutoCreateCollection(true);
config.setDefaultCollectionName("test");
PgvectorVectorStore store = new PgvectorVectorStore(config);
StoreResult update = store.delete("145314895749100ae8306079519b3393","e83518d36b6d5de8199b40e3ef4e4ce1");
System.out.println(update);
}
}

View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-qcloud</name>
<artifactId>easy-agents-store-qcloud</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,177 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.qcloud;
import com.easyagents.core.document.Document;
import com.easyagents.core.model.client.HttpClient;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.StringUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* doc https://cloud.tencent.com/document/product/1709/95121
*/
public class QCloudVectorStore extends DocumentStore {
private QCloudVectorStoreConfig config;
private final HttpClient httpUtil = new HttpClient();
public QCloudVectorStore(QCloudVectorStoreConfig config) {
this.config = config;
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("database", config.getDatabase());
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
List<Map<String, Object>> payloadDocs = new ArrayList<>();
for (Document vectorDocument : documents) {
Map<String, Object> document = new HashMap<>();
if (vectorDocument.getMetadataMap() != null) {
document.putAll(vectorDocument.getMetadataMap());
}
document.put("vector", vectorDocument.getVector());
document.put("id", vectorDocument.getId());
payloadDocs.add(document);
}
payloadMap.put("documents", payloadDocs);
String payload = JSON.toJSONString(payloadMap);
httpUtil.post(config.getHost() + "/document/upsert", headers, payload);
return StoreResult.successWithIds(documents);
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("database", config.getDatabase());
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
Map<String, Object> documentIdsObj = new HashMap<>();
documentIdsObj.put("documentIds", ids);
payloadMap.put("query", documentIdsObj);
String payload = JSON.toJSONString(payloadMap);
httpUtil.post(config.getHost() + "/document/delete", headers, payload);
return StoreResult.success();
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("database", config.getDatabase());
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
for (Document document : documents) {
Map<String, Object> documentIdsObj = new HashMap<>();
documentIdsObj.put("documentIds", Collections.singletonList(document.getId()));
payloadMap.put("query", documentIdsObj);
payloadMap.put("update", document.getMetadataMap());
String payload = JSON.toJSONString(payloadMap);
httpUtil.post(config.getHost() + "/document/update", headers, payload);
}
return StoreResult.successWithIds(documents);
}
@Override
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
Map<String, Object> payloadMap = new HashMap<>();
payloadMap.put("database", config.getDatabase());
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
Map<String, Object> searchMap = new HashMap<>();
searchMap.put("vectors", Collections.singletonList(searchWrapper.getVector()));
if (searchWrapper.getMaxResults() != null) {
searchMap.put("limit", searchWrapper.getMaxResults());
}
payloadMap.put("search", searchMap);
String payload = JSON.toJSONString(payloadMap);
// https://cloud.tencent.com/document/product/1709/95123
String response = httpUtil.post(config.getHost() + "/document/search", headers, payload);
if (StringUtil.noText(response)) {
return null;
}
List<Document> result = new ArrayList<>();
JSONObject rootObject = JSON.parseObject(response);
int code = rootObject.getIntValue("code");
if (code != 0) {
LoggerFactory.getLogger(QCloudVectorStore.class).error("can not search in QCloudVectorStore, code:" + code + ", message: " + rootObject.getString("msg"));
return null;
}
JSONArray rootDocs = rootObject.getJSONArray("documents");
for (int i = 0; i < rootDocs.size(); i++) {
JSONArray docs = rootDocs.getJSONArray(i);
for (int j = 0; j < docs.size(); j++) {
JSONObject doc = docs.getJSONObject(j);
Document vd = new Document();
vd.setId(doc.getString("id"));
doc.remove("id");
vd.addMetadata(doc);
result.add(vd);
}
}
return result;
}
}

View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.qcloud;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
public class QCloudVectorStoreConfig implements DocumentStoreConfig {
private String host;
private String apiKey;
private String account;
private String database;
private String defaultCollectionName;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public String getAccount() {
return account;
}
public void setAccount(String account) {
this.account = account;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.host, this.apiKey, this.account, this.database, this.defaultCollectionName);
}
}

View File

@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-qdrant</name>
<artifactId>easy-agents-store-qdrant</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>1.65.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>1.65.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<version>1.65.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.qdrant</groupId>
<artifactId>client</artifactId>
<version>1.14.0</version>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,186 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.qdrant;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.CollectionUtil;
import com.easyagents.core.util.StringUtil;
import io.grpc.Grpc;
import io.grpc.ManagedChannel;
import io.grpc.TlsChannelCredentials;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections;
import io.qdrant.client.grpc.JsonWithInt;
import io.qdrant.client.grpc.Points;
import io.qdrant.client.grpc.Points.*;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import static io.qdrant.client.ConditionFactory.matchKeyword;
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorsFactory.vectors;
import static io.qdrant.client.WithPayloadSelectorFactory.enable;
public class QdrantVectorStore extends DocumentStore {
private final QdrantVectorStoreConfig config;
private final QdrantClient client;
private final String defaultCollectionName;
private boolean isCreateCollection = false;
public QdrantVectorStore(QdrantVectorStoreConfig config) throws IOException {
this.config = config;
this.defaultCollectionName = config.getDefaultCollectionName();
String uri = config.getUri();
int port = 6334;
QdrantGrpcClient.Builder builder;
if (StringUtil.hasText(config.getCaPath())) {
ManagedChannel channel = Grpc.newChannelBuilder(
uri,
TlsChannelCredentials.newBuilder().trustManager(new File(config.getCaPath())).build()
).build();
builder = QdrantGrpcClient.newBuilder(channel, true);
} else {
if (uri.contains(":")) {
uri = uri.split(":")[0];
port = Integer.parseInt(uri.split(":")[1]);
}
builder = QdrantGrpcClient.newBuilder(uri, port, false);
}
if (StringUtil.hasText(config.getApiKey())) {
builder.withApiKey(config.getApiKey());
}
this.client = new QdrantClient(builder.build());
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
List<PointStruct> points = new ArrayList<>();
int size = 1024;
for (Document doc : documents) {
size = doc.getVector().length;
Map<String, JsonWithInt.Value> payload = new HashMap<>();
payload.put("content", value(doc.getContent()));
points.add(PointStruct.newBuilder()
.setId(id(Long.parseLong(doc.getId().toString())))
.setVectors(vectors(doc.getVector()))
.putAllPayload(payload)
.build());
}
try {
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
if (config.isAutoCreateCollection() && !isCreateCollection) {
Boolean exists = client.collectionExistsAsync(collectionName).get();
if (!exists) {
client.createCollectionAsync(collectionName, Collections.VectorParams.newBuilder()
.setDistance(Collections.Distance.Cosine)
.setSize(size)
.build())
.get();
}
} else {
isCreateCollection = true;
}
if (CollectionUtil.hasItems(points)) {
client.upsertAsync(collectionName, points).get();
}
return StoreResult.successWithIds(documents);
} catch (Exception e) {
return StoreResult.fail();
}
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
try {
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
List<PointId> pointIds = ids.stream()
.map(id -> id((Long) id))
.collect(Collectors.toList());
client.deleteAsync(collectionName, pointIds).get();
return StoreResult.success();
} catch (Exception e) {
return StoreResult.fail();
}
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
try {
List<PointStruct> points = new ArrayList<>();
for (Document doc : documents) {
Map<String, JsonWithInt.Value> payload = new HashMap<>();
payload.put("content", value(doc.getContent()));
points.add(PointStruct.newBuilder()
.setId(id(Long.parseLong(doc.getId().toString())))
.setVectors(vectors(doc.getVector()))
.putAllPayload(payload)
.build());
}
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
if (CollectionUtil.hasItems(points)) {
client.upsertAsync(collectionName, points).get();
}
return StoreResult.successWithIds(documents);
} catch (Exception e) {
return StoreResult.fail();
}
}
@Override
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
List<Document> documents = new ArrayList<>();
try {
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
QueryPoints.Builder query = QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setLimit(wrapper.getMaxResults())
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(true).build())
.setWithPayload(enable(true));
if (wrapper.getVector() != null) {
query.setQuery(nearest(wrapper.getVector()));
}
if (StringUtil.hasText(wrapper.getText())) {
query.setFilter(Filter.newBuilder().addMust(matchKeyword("content", wrapper.getText())));
}
List<ScoredPoint> data = client.queryAsync(query.build()).get();
for (ScoredPoint point : data) {
Document doc = new Document();
doc.setId(point.getId().getNum());
doc.setVector(point.getVectors().getVector().getDataList());
doc.setContent(point.getPayloadMap().get("content").getStringValue());
documents.add(doc);
}
return documents;
} catch (Exception e) {
return documents;
}
}
public QdrantClient getClient() {
return client;
}
}

View File

@@ -0,0 +1,73 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.qdrant;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
public class QdrantVectorStoreConfig implements DocumentStoreConfig {
private String uri;
private String caPath;
private String defaultCollectionName;
private String apiKey;
private boolean autoCreateCollection = true;
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
public String getCaPath() {
return caPath;
}
public void setCaPath(String caPath) {
this.caPath = caPath;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public boolean isAutoCreateCollection() {
return autoCreateCollection;
}
public void setAutoCreateCollection(boolean autoCreateCollection) {
this.autoCreateCollection = autoCreateCollection;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.uri);
}
}

View File

@@ -0,0 +1,65 @@
package com.easyagents.store.qdrant;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class QdrantVectorStoreTest {
@Test
public void testSaveVectors() throws Exception {
QdrantVectorStore db = getDb();
StoreOptions options = new StoreOptions();
options.setCollectionName("test_collection1");
List<Document> list = new ArrayList<>();
Document doc1 = new Document();
doc1.setId(1L);
doc1.setContent("test1");
doc1.setVector(new float[]{5.2f, 4.4f});
list.add(doc1);
Document doc2 = new Document();
doc2.setId(2L);
doc2.setContent("test2");
doc2.setVector(new float[]{5.2f, 3.9f});
list.add(doc2);
Document doc3 = new Document();
doc3.setId(3);
doc3.setContent("test3");
doc3.setVector(new float[]{4.9f, 3.4f});
list.add(doc3);
db.store(list, options);
}
@Test
public void testQuery() throws Exception {
QdrantVectorStore db = getDb();;
StoreOptions options = new StoreOptions();
options.setCollectionName("test_collection1");
SearchWrapper search = new SearchWrapper();
search.setVector(new float[]{5.2f, 3.9f});
//search.setText("test1");
search.setMaxResults(1);
List<Document> record = db.search(search, options);
System.out.println(record);
}
@Test
public void testDelete() throws Exception {
QdrantVectorStore db = getDb();
StoreOptions options = new StoreOptions();
options.setCollectionName("test_collection1");
db.delete(Collections.singletonList(3L), options);
}
private QdrantVectorStore getDb() throws Exception {
QdrantVectorStoreConfig config = new QdrantVectorStoreConfig();
config.setUri("localhost");
config.setDefaultCollectionName("test_collection1");
return new QdrantVectorStore(config);
}
}

View File

@@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-redis</name>
<artifactId>easy-agents-store-redis</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>5.2.0</version>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,256 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.redis;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.StringUtil;
import com.alibaba.fastjson2.JSON;
import kotlin.collections.ArrayDeque;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPooled;
import redis.clients.jedis.Pipeline;
import redis.clients.jedis.json.Path2;
import redis.clients.jedis.search.FTCreateParams;
import redis.clients.jedis.search.IndexDataType;
import redis.clients.jedis.search.Query;
import redis.clients.jedis.search.SearchResult;
import redis.clients.jedis.search.schemafields.SchemaField;
import redis.clients.jedis.search.schemafields.TextField;
import redis.clients.jedis.search.schemafields.VectorField;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.*;
public class RedisVectorStore extends DocumentStore {
protected final RedisVectorStoreConfig config;
protected final JedisPooled jedis;
protected final Set<String> redisIndexesCache = new HashSet<>();
protected static final Logger logger = LoggerFactory.getLogger(RedisVectorStore.class);
public RedisVectorStore(RedisVectorStoreConfig config) {
this.config = config;
this.jedis = new JedisPooled(
URI.create(config.getUri())
);
}
protected void createSchemaIfNecessary(String indexName) {
if (redisIndexesCache.contains(indexName)) {
return;
}
// 检查 indexName 是否存在
Set<String> existIndexes = this.jedis.ftList();
if (existIndexes != null && existIndexes.contains(indexName)) {
redisIndexesCache.add(indexName);
return;
}
FTCreateParams ftCreateParams = FTCreateParams.createParams()
.on(IndexDataType.JSON)
.addPrefix(getPrefix(indexName));
jedis.ftCreate(indexName, ftCreateParams, schemaFields());
redisIndexesCache.add(indexName);
}
protected Iterable<SchemaField> schemaFields() {
Map<String, Object> vectorAttrs = new HashMap<>();
//支持 COSINE: 余弦距离 , IP: 内积距离, L2: 欧几里得距离
vectorAttrs.put("DISTANCE_METRIC", "COSINE");
vectorAttrs.put("TYPE", "FLOAT32");
vectorAttrs.put("DIM", this.getEmbeddingModel().dimensions());
List<SchemaField> fields = new ArrayList<>();
fields.add(TextField.of(jsonPath("text")).as("text").weight(1.0));
fields.add(VectorField.builder()
.fieldName(jsonPath("vector"))
.algorithm(VectorField.VectorAlgorithm.HNSW)
.attributes(vectorAttrs)
.as("vector")
.build());
return fields;
}
protected String jsonPath(String field) {
return "$." + field;
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
String indexName = createIndexName(options);
if (StringUtil.noText(indexName)) {
throw new IllegalStateException("IndexName is null or blank. please config the \"defaultCollectionName\" or store with designative collectionName.");
}
createSchemaIfNecessary(indexName);
try (Pipeline pipeline = jedis.pipelined();) {
for (Document document : documents) {
java.util.Map<String, Object> fields = new HashMap<>();
fields.put("text", document.getContent());
fields.put("vector", document.getVector());
//put all metadata
Map<String, Object> metadataMap = document.getMetadataMap();
if (metadataMap != null) {
fields.putAll(metadataMap);
}
String key = getPrefix(indexName) + document.getId();
pipeline.jsonSetWithEscape(key, Path2.of("$"), fields);
}
List<Object> objects = pipeline.syncAndReturnAll();
for (Object object : objects) {
if (!object.equals("OK")) {
logger.error("Could not store document: {}", object);
return StoreResult.fail();
}
}
}
return StoreResult.successWithIds(documents);
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
String indexName = createIndexName(options);
try (Pipeline pipeline = this.jedis.pipelined()) {
for (Object id : ids) {
String key = getPrefix(indexName) + id;
pipeline.jsonDel(key);
}
List<Object> objects = pipeline.syncAndReturnAll();
for (Object object : objects) {
if (!object.equals(1L)) {
logger.error("Could not delete document: {}", object);
return StoreResult.fail();
}
}
}
return StoreResult.success();
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
return doStore(documents, options);
}
@Override
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
String indexName = createIndexName(options);
if (StringUtil.noText(indexName)) {
throw new IllegalStateException("IndexName is null or blank. please config the \"defaultCollectionName\" or store with designative collectionName.");
}
createSchemaIfNecessary(indexName);
// 创建查询向量
byte[] vectorBytes = new byte[wrapper.getVector().length * 4];
FloatBuffer floatBuffer = ByteBuffer.wrap(vectorBytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer();
for (Float v : wrapper.getVector()) {
floatBuffer.put(v);
}
List<String> returnFields = new ArrayList<>();
returnFields.add("text");
returnFields.add("vector");
returnFields.add("score");
if (wrapper.getOutputFields() != null) {
returnFields.addAll(wrapper.getOutputFields());
}
// 使用 KNN 算法进行向量相似度搜索
Query query = new Query("*=>[KNN " + wrapper.getMaxResults() + " @vector $BLOB AS score]")
.addParam("BLOB", vectorBytes)
.returnFields(returnFields.toArray(new String[0]))
.setSortBy("score", true)
.limit(0, wrapper.getMaxResults())
.dialect(2);
int keyPrefixLen = this.getPrefix(indexName).length();
// 执行搜索
SearchResult searchResult = jedis.ftSearch(indexName, query);
List<redis.clients.jedis.search.Document> searchDocuments = searchResult.getDocuments();
List<Document> documents = new ArrayDeque<>(searchDocuments.size());
for (redis.clients.jedis.search.Document document : searchDocuments) {
String id = document.getId().substring(keyPrefixLen);
Document doc = new Document();
doc.setId(id);
doc.setContent(document.getString("text"));
Object vector = document.get("vector");
if (vector != null) {
float[] doubles = JSON.parseObject(vector.toString(), float[].class);
doc.setVector(doubles);
}
if (wrapper.getOutputFields() != null) {
for (String field : wrapper.getOutputFields()) {
doc.addMetadata(field, document.getString(field));
}
}
double distance = 1.0d - similarityScore(document);
// 相似度得分设置为0-1 0表示最不相似 1表示最相似
doc.setScore(1.0d - distance);
documents.add(doc);
}
return documents;
}
protected float similarityScore(redis.clients.jedis.search.Document doc) {
return (2 - Float.parseFloat(doc.getString("score"))) / 2;
}
protected String createIndexName(StoreOptions options) {
return options.getCollectionNameOrDefault(config.getDefaultCollectionName());
}
@NotNull
protected String getPrefix(String indexName) {
return this.config.getStorePrefix() + indexName + ":";
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.redis;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
public class RedisVectorStoreConfig implements DocumentStoreConfig {
private String uri;
private String storePrefix = "docs:";
private String defaultCollectionName;
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
public String getStorePrefix() {
return storePrefix;
}
public void setStorePrefix(String storePrefix) {
this.storePrefix = storePrefix;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.uri);
}
}

View File

@@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-vectorex</name>
<artifactId>easy-agents-store-vectorex</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.github.javpower</groupId>
<artifactId>vectorex-client</artifactId>
<version>1.5.3</version>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,158 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.vectorex;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import io.github.javpower.vectorexclient.VectorRexClient;
import io.github.javpower.vectorexclient.builder.QueryBuilder;
import io.github.javpower.vectorexclient.entity.MetricType;
import io.github.javpower.vectorexclient.entity.ScalarField;
import io.github.javpower.vectorexclient.entity.VectoRexEntity;
import io.github.javpower.vectorexclient.entity.VectorFiled;
import io.github.javpower.vectorexclient.req.CollectionDataAddReq;
import io.github.javpower.vectorexclient.req.CollectionDataDelReq;
import io.github.javpower.vectorexclient.req.VectoRexCollectionReq;
import io.github.javpower.vectorexclient.res.DbData;
import io.github.javpower.vectorexclient.res.ServerResponse;
import io.github.javpower.vectorexclient.res.VectorSearchResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class VectoRexStore extends DocumentStore {
private final VectoRexStoreConfig config;
private final VectorRexClient client;
private final String defaultCollectionName;
private boolean isCreateCollection = false;
private static final Logger logger = LoggerFactory.getLogger(VectoRexStore.class);
public VectoRexStore(VectoRexStoreConfig config) {
this.config = config;
this.defaultCollectionName = config.getDefaultCollectionName();
this.client = new VectorRexClient(config.getUri(), config.getUsername(), config.getPassword());
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
List<Map<String, Object>> data = new ArrayList<>();
for (Document doc : documents) {
Map<String, Object> dict = new HashMap<>();
dict.put("id", String.valueOf(doc.getId()));
dict.put("content", doc.getContent());
dict.put("vector", doc.getVectorAsList());
data.add(dict);
}
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
if (config.isAutoCreateCollection() && !isCreateCollection) {
ServerResponse<List<VectoRexEntity>> collections = client.getCollections();
if (collections.getData() == null || collections.getData().stream().noneMatch(e -> e.getCollectionName().equals(collectionName))) {
createCollection(collectionName);
} else {
isCreateCollection = true;
}
}
for (Map<String, Object> map : data) {
CollectionDataAddReq req = CollectionDataAddReq.builder().collectionName(collectionName).metadata(map).build();
client.addCollectionData(req);
}
return StoreResult.successWithIds(documents);
}
private Boolean createCollection(String collectionName) {
List<ScalarField> scalarFields = new ArrayList();
ScalarField id = ScalarField.builder().name("id").isPrimaryKey(true).build();
ScalarField content = ScalarField.builder().name("content").isPrimaryKey(false).build();
scalarFields.add(id);
scalarFields.add(content);
List<VectorFiled> vectorFiles = new ArrayList();
VectorFiled vector = VectorFiled.builder().name("vector").metricType(MetricType.FLOAT_COSINE_DISTANCE).dimensions(this.getEmbeddingModel().dimensions()).build();
vectorFiles.add(vector);
ServerResponse<Void> response = client.createCollection(VectoRexCollectionReq.builder().collectionName(collectionName).scalarFields(scalarFields).vectorFileds(vectorFiles).build());
return response.isSuccess();
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
for (Object id : ids) {
CollectionDataDelReq req = CollectionDataDelReq.builder().collectionName(options.getCollectionNameOrDefault(defaultCollectionName)).id((String) id).build();
ServerResponse<Void> response = client.deleteCollectionData(req);
if (!response.isSuccess()) {
return StoreResult.fail();
}
}
return StoreResult.success();
}
@Override
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
ServerResponse<List<VectorSearchResult>> response = client.queryCollectionData(QueryBuilder.lambda(options.getCollectionNameOrDefault(defaultCollectionName))
.vector("vector", Collections.singletonList(searchWrapper.getVectorAsList())).topK(searchWrapper.getMaxResults()));
if (!response.isSuccess()) {
logger.error("Error searching in VectoRex", response.getMsg());
return Collections.emptyList();
}
List<VectorSearchResult> data = response.getData();
List<Document> documents = new ArrayList<>();
for (VectorSearchResult result : data) {
DbData dd = result.getData();
Map<String, Object> metadata = dd.getMetadata();
Document doc = new Document();
doc.setId(result.getId());
doc.setContent((String) metadata.get("content"));
Object vectorObj = metadata.get("vector");
if (vectorObj instanceof List) {
//noinspection unchecked
doc.setVector((List<Float>) vectorObj);
}
documents.add(doc);
}
return documents;
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
List<Map<String, Object>> data = new ArrayList<>();
for (Document doc : documents) {
Map<String, Object> dict = new HashMap<>();
dict.put("id", String.valueOf(doc.getId()));
dict.put("content", doc.getContent());
dict.put("vector", doc.getVectorAsList());
data.add(dict);
}
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
for (Map<String, Object> map : data) {
CollectionDataAddReq req = CollectionDataAddReq.builder().collectionName(collectionName).metadata(map).build();
client.updateCollectionData(req);
}
return StoreResult.successWithIds(documents);
}
public VectorRexClient getClient() {
return client;
}
}

View File

@@ -0,0 +1,76 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.vectorex;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
public class VectoRexStoreConfig implements DocumentStoreConfig {
private String uri;
private String username;
private String password;
private String defaultCollectionName;
private boolean autoCreateCollection = true;
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
public boolean isAutoCreateCollection() {
return autoCreateCollection;
}
public void setAutoCreateCollection(boolean autoCreateCollection) {
this.autoCreateCollection = autoCreateCollection;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.uri);
}
}

View File

@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-store</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store-vectorexdb</name>
<artifactId>easy-agents-store-vectorexdb</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-core</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.github.javpower</groupId>
<artifactId>vectorex-core</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-chat-openai</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,156 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.vectorex;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.core.util.CollectionUtil;
import com.google.common.collect.Lists;
import io.github.javpower.vectorex.keynote.core.DbData;
import io.github.javpower.vectorex.keynote.core.VectorData;
import io.github.javpower.vectorex.keynote.core.VectorSearchResult;
import io.github.javpower.vectorex.keynote.model.MetricType;
import io.github.javpower.vectorex.keynote.model.VectorFiled;
import io.github.javpower.vectorexcore.VectoRexClient;
import io.github.javpower.vectorexcore.entity.KeyValue;
import io.github.javpower.vectorexcore.entity.ScalarField;
import io.github.javpower.vectorexcore.entity.VectoRexEntity;
import java.util.*;
public class VectoRexStore extends DocumentStore {
private final VectoRexStoreConfig config;
private final VectoRexClient client;
private final String defaultCollectionName;
private boolean isCreateCollection=false;
public VectoRexStore(VectoRexStoreConfig config) {
this.config = config;
this.defaultCollectionName=config.getDefaultCollectionName();
this.client = new VectoRexClient(config.getUri());
}
@Override
public StoreResult doStore(List<Document> documents, StoreOptions options) {
List<DbData> data=new ArrayList<>();
for (Document doc : documents) {
Map<String, Object> dict=new HashMap<>();
dict.put("id",String.valueOf(doc.getId()));
dict.put("content", doc.getContent());
dict.put("vector", doc.getVector());
DbData dbData=new DbData();
dbData.setId(String.valueOf(doc.getId()));
dbData.setMetadata(dict);
VectorData vd=new VectorData(dbData.getId(),doc.getVector());
vd.setName("vector");
dbData.setVectorFiled(Lists.newArrayList(vd));
data.add(dbData);
}
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
if(config.isAutoCreateCollection()&&!isCreateCollection){
List<VectoRexEntity> collections = client.getCollections();
if(CollectionUtil.noItems(collections)||collections.stream().noneMatch(e -> e.getCollectionName().equals(collectionName))){
createCollection(collectionName);
}else {
isCreateCollection=true;
}
}
if(CollectionUtil.hasItems(data)){
client.getStore(collectionName).saveAll(data);
}
return StoreResult.successWithIds(documents);
}
private void createCollection(String collectionName) {
VectorFiled vectorFiled = new VectorFiled();
vectorFiled.setDimensions(this.getEmbeddingModel().dimensions());
vectorFiled.setName("vector");
vectorFiled.setMetricType(MetricType.FLOAT_COSINE_DISTANCE);
VectoRexEntity entity=new VectoRexEntity();
entity.setCollectionName(collectionName);
List<KeyValue<String, VectorFiled>> vectorFiles=new ArrayList<>();
vectorFiles.add(new KeyValue<>("vector",vectorFiled));
List<KeyValue<String, ScalarField>> scalarFields=new ArrayList<>();
ScalarField id = new ScalarField();
id.setName("id");
id.setIsPrimaryKey(true);
scalarFields.add(new KeyValue<>("id",id));
ScalarField content = new ScalarField();
content.setName("content");
content.setIsPrimaryKey(false);
scalarFields.add(new KeyValue<>("content",content));
entity.setVectorFileds(vectorFiles);
entity.setScalarFields(scalarFields);
client.createCollection(entity);
}
@Override
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).deleteAll((List<String>) ids);
return StoreResult.success();
}
@Override
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
List<VectorSearchResult> data = client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).search("vector", searchWrapper.getVectorAsList(), searchWrapper.getMaxResults(), null);
List<Document> documents = new ArrayList<>();
for (VectorSearchResult result : data) {
DbData dd = result.getData();
Map<String, Object> metadata = dd.getMetadata();
Document doc=new Document();
doc.setId(result.getId());
doc.setContent((String) metadata.get("content"));
Object vectorObj = metadata.get("vector");
if (vectorObj instanceof List) {
//noinspection unchecked
doc.setVector((List<Float>) vectorObj);
}
documents.add(doc);
}
return documents;
}
@Override
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
if (documents == null || documents.isEmpty()) {
return StoreResult.success();
}
for (Document doc : documents) {
Map<String, Object> dict=new HashMap<>();
dict.put("id",String.valueOf(doc.getId()));
dict.put("content", doc.getContent());
dict.put("vector", doc.getVector());
DbData dbData=new DbData();
dbData.setId(String.valueOf(doc.getId()));
dbData.setMetadata(dict);
VectorData vd=new VectorData(dbData.getId(),doc.getVector());
vd.setName("vector");
dbData.setVectorFiled(Lists.newArrayList(vd));
client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).update(dbData);
}
return StoreResult.successWithIds(documents);
}
public VectoRexClient getClient() {
return client;
}
}

View File

@@ -0,0 +1,56 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.vectorex;
import com.easyagents.core.store.DocumentStoreConfig;
import com.easyagents.core.util.StringUtil;
public class VectoRexStoreConfig implements DocumentStoreConfig {
private String uri;
private String defaultCollectionName;
private boolean autoCreateCollection = true;
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
public String getDefaultCollectionName() {
return defaultCollectionName;
}
public void setDefaultCollectionName(String defaultCollectionName) {
this.defaultCollectionName = defaultCollectionName;
}
public boolean isAutoCreateCollection() {
return autoCreateCollection;
}
public void setAutoCreateCollection(boolean autoCreateCollection) {
this.autoCreateCollection = autoCreateCollection;
}
@Override
public boolean checkAvailable() {
return StringUtil.hasText(this.uri);
}
}

View File

@@ -0,0 +1,63 @@
/*
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.easyagents.store.vectorex.test;
import com.easyagents.core.document.Document;
import com.easyagents.core.model.chat.ChatModel;
import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreResult;
import com.easyagents.llm.openai.OpenAIChatModel;
import com.easyagents.llm.openai.OpenAIChatConfig;
import com.easyagents.store.vectorex.VectoRexStore;
import com.easyagents.store.vectorex.VectoRexStoreConfig;
import java.util.List;
public class Test {
public static void main(String[] args) {
OpenAIChatConfig openAILlmConfig= new OpenAIChatConfig();
openAILlmConfig.setApiKey("");
openAILlmConfig.setEndpoint("");
openAILlmConfig.setModel("");
ChatModel chatModel = new OpenAIChatModel(openAILlmConfig);
VectoRexStoreConfig config = new VectoRexStoreConfig();
config.setDefaultCollectionName("test05");
VectoRexStore store = new VectoRexStore(config);
// store.setEmbeddingModel(chatModel);
Document document = new Document();
document.setContent("你好");
document.setId(1);
store.store(document);
SearchWrapper sw = new SearchWrapper();
sw.setText("你好");
List<Document> search = store.search(sw);
System.out.println(search);
StoreResult result = store.delete("1");
System.out.println("-------delete-----" + result);
search = store.search(sw);
System.out.println(search);
}
}

35
easy-agents-store/pom.xml Normal file
View File

@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-parent</artifactId>
<version>${revision}</version>
</parent>
<name>easy-agents-store</name>
<artifactId>easy-agents-store</artifactId>
<packaging>pom</packaging>
<modules>
<module>easy-agents-store-qcloud</module>
<module>easy-agents-store-aliyun</module>
<module>easy-agents-store-chroma</module>
<module>easy-agents-store-redis</module>
<module>easy-agents-store-opensearch</module>
<module>easy-agents-store-elasticsearch</module>
<module>easy-agents-store-vectorex</module>
<module>easy-agents-store-vectorexdb</module>
<module>easy-agents-store-pgvector</module>
<module>easy-agents-store-qdrant</module>
</modules>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
</project>