初始化
This commit is contained in:
35
easy-agents-store/easy-agents-store-aliyun/pom.xml
Normal file
35
easy-agents-store/easy-agents-store-aliyun/pom.xml
Normal file
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-aliyun</name>
|
||||
<artifactId>easy-agents-store-aliyun</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.aliyun;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.model.client.HttpClient;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 文档 https://help.aliyun.com/document_detail/2510317.html
|
||||
*/
|
||||
public class AliyunVectorStore extends DocumentStore {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AliyunVectorStore.class);
|
||||
|
||||
private AliyunVectorStoreConfig config;
|
||||
|
||||
private final HttpClient httpUtil = new HttpClient();
|
||||
|
||||
public AliyunVectorStore(AliyunVectorStoreConfig config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("dashvector-auth-token", config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
|
||||
List<Map<String, Object>> payloadDocs = new ArrayList<>();
|
||||
for (Document vectorDocument : documents) {
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
if (vectorDocument.getMetadataMap() != null) {
|
||||
document.put("fields", vectorDocument.getMetadataMap());
|
||||
}
|
||||
document.put("vector", vectorDocument.getVector());
|
||||
document.put("id", vectorDocument.getId());
|
||||
payloadDocs.add(document);
|
||||
}
|
||||
|
||||
payloadMap.put("docs", payloadDocs);
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
String url = "https://" + config.getEndpoint() + "/v1/collections/"
|
||||
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
|
||||
String response = httpUtil.post(url, headers, payload);
|
||||
|
||||
if (StringUtil.noText(response)) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
JSONObject jsonObject = JSON.parseObject(response);
|
||||
Integer code = jsonObject.getInteger("code");
|
||||
String message = jsonObject.getString("message");
|
||||
|
||||
if (code != null && code == 0 && "Success".equals(message)) {
|
||||
|
||||
return StoreResult.successWithIds(documents);
|
||||
} else {
|
||||
LOG.error("delete vector fail: " + response);
|
||||
return StoreResult.fail(message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("dashvector-auth-token", config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("ids", ids);
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
|
||||
String url = "https://" + config.getEndpoint() + "/v1/collections/"
|
||||
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
|
||||
String response = httpUtil.delete(url, headers, payload);
|
||||
if (StringUtil.noText(response)) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
JSONObject jsonObject = JSON.parseObject(response);
|
||||
Integer code = jsonObject.getInteger("code");
|
||||
if (code != null && code == 0) {
|
||||
return StoreResult.success();
|
||||
} else {
|
||||
LOG.error("delete vector fail: " + response);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("dashvector-auth-token", config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
|
||||
List<Map<String, Object>> payloadDocs = new ArrayList<>();
|
||||
for (Document vectorDocument : documents) {
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
if (vectorDocument.getMetadataMap() != null) {
|
||||
document.put("fields", vectorDocument.getMetadataMap());
|
||||
}
|
||||
document.put("vector", vectorDocument.getVector());
|
||||
document.put("id", vectorDocument.getId());
|
||||
payloadDocs.add(document);
|
||||
}
|
||||
|
||||
payloadMap.put("docs", payloadDocs);
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
String url = "https://" + config.getEndpoint() + "/v1/collections/"
|
||||
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/docs";
|
||||
String response = httpUtil.put(url, headers, payload);
|
||||
|
||||
if (StringUtil.noText(response)) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
JSONObject jsonObject = JSON.parseObject(response);
|
||||
Integer code = jsonObject.getInteger("code");
|
||||
if (code != null && code == 0) {
|
||||
return StoreResult.successWithIds(documents);
|
||||
} else {
|
||||
LOG.error("delete vector fail: " + response);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("dashvector-auth-token", config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("vector", wrapper.getVector());
|
||||
payloadMap.put("topk", wrapper.getMaxResults());
|
||||
payloadMap.put("include_vector", wrapper.isWithVector());
|
||||
payloadMap.put("filter", wrapper.toFilterExpression());
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
String url = "https://" + config.getEndpoint() + "/v1/collections/"
|
||||
+ options.getCollectionNameOrDefault(config.getDefaultCollectionName()) + "/query";
|
||||
String result = httpUtil.post(url, headers, payload);
|
||||
|
||||
if (StringUtil.noText(result)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//https://help.aliyun.com/document_detail/2510319.html
|
||||
JSONObject rootObject = JSON.parseObject(result);
|
||||
int code = rootObject.getIntValue("code");
|
||||
if (code != 0) {
|
||||
//error
|
||||
LoggerFactory.getLogger(AliyunVectorStore.class).error("can not search data AliyunVectorStore(code: " + code + "), message: " + rootObject.getString("message"));
|
||||
return null;
|
||||
}
|
||||
|
||||
JSONArray output = rootObject.getJSONArray("output");
|
||||
List<Document> documents = new ArrayList<>(output.size());
|
||||
for (int i = 0; i < output.size(); i++) {
|
||||
JSONObject jsonObject = output.getJSONObject(i);
|
||||
Document document = new Document();
|
||||
document.setId(jsonObject.getString("id"));
|
||||
document.setVector(jsonObject.getObject("vector", float[].class));
|
||||
// 阿里云数据采用余弦相似度计算 jsonObject.getDoubleValue("score") 表示余弦距离,
|
||||
// 原始余弦距离范围是[0, 2],0表示最相似,2表示最不相似
|
||||
Double distance = jsonObject.getDouble("score");
|
||||
if (distance != null) {
|
||||
double score = distance / 2.0;
|
||||
document.setScore(1.0d - score);
|
||||
}
|
||||
|
||||
|
||||
JSONObject fields = jsonObject.getJSONObject("fields");
|
||||
document.addMetadata(fields);
|
||||
|
||||
documents.add(document);
|
||||
}
|
||||
|
||||
return documents;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.aliyun;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
/**
|
||||
* https://help.aliyun.com/document_detail/2510317.html
|
||||
*/
|
||||
public class AliyunVectorStoreConfig implements DocumentStoreConfig {
|
||||
private String endpoint;
|
||||
private String apiKey;
|
||||
private String database;
|
||||
private String defaultCollectionName;
|
||||
|
||||
|
||||
public String getEndpoint() {
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
public void setEndpoint(String endpoint) {
|
||||
this.endpoint = endpoint;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.endpoint, this.apiKey, this.database, this.defaultCollectionName);
|
||||
}
|
||||
}
|
||||
47
easy-agents-store/easy-agents-store-chroma/pom.xml
Normal file
47
easy-agents-store/easy-agents-store-chroma/pom.xml
Normal file
@@ -0,0 +1,47 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-chroma</name>
|
||||
<artifactId>easy-agents-store-chroma</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
<version>2.10.1</version>
|
||||
</dependency> -->
|
||||
|
||||
<!-- 测试依赖 -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.chroma;
|
||||
|
||||
import com.easyagents.core.store.condition.Condition;
|
||||
import com.easyagents.core.store.condition.ConditionType;
|
||||
import com.easyagents.core.store.condition.ExpressionAdaptor;
|
||||
import com.easyagents.core.store.condition.Value;
|
||||
|
||||
import java.util.StringJoiner;
|
||||
|
||||
public class ChromaExpressionAdaptor implements ExpressionAdaptor {
|
||||
|
||||
public static final ChromaExpressionAdaptor DEFAULT = new ChromaExpressionAdaptor();
|
||||
|
||||
@Override
|
||||
public String toOperationSymbol(ConditionType type) {
|
||||
if (type == ConditionType.EQ) {
|
||||
return " == ";
|
||||
} else if (type == ConditionType.NE) {
|
||||
return " != ";
|
||||
} else if (type == ConditionType.GT) {
|
||||
return " > ";
|
||||
} else if (type == ConditionType.GE) {
|
||||
return " >= ";
|
||||
} else if (type == ConditionType.LT) {
|
||||
return " < ";
|
||||
} else if (type == ConditionType.LE) {
|
||||
return " <= ";
|
||||
} else if (type == ConditionType.IN) {
|
||||
return " IN ";
|
||||
}
|
||||
return type.getDefaultSymbol();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toCondition(Condition condition) {
|
||||
if (condition.getType() == ConditionType.BETWEEN) {
|
||||
Object[] values = (Object[]) ((Value) condition.getRight()).getValue();
|
||||
return "(" + toLeft(condition.getLeft())
|
||||
+ toOperationSymbol(ConditionType.GE)
|
||||
+ values[0] + " && "
|
||||
+ toLeft(condition.getLeft())
|
||||
+ toOperationSymbol(ConditionType.LE)
|
||||
+ values[1] + ")";
|
||||
}
|
||||
|
||||
return ExpressionAdaptor.super.toCondition(condition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toValue(Condition condition, Object value) {
|
||||
if (value == null) {
|
||||
return "null";
|
||||
}
|
||||
|
||||
if (condition.getType() == ConditionType.IN) {
|
||||
Object[] values = (Object[]) value;
|
||||
StringJoiner stringJoiner = new StringJoiner(",", "[", "]");
|
||||
for (Object v : values) {
|
||||
if (v != null) {
|
||||
stringJoiner.add("\"" + v + "\"");
|
||||
}
|
||||
}
|
||||
return stringJoiner.toString();
|
||||
} else if (value instanceof String) {
|
||||
return "\"" + value + "\"";
|
||||
} else if (value instanceof Boolean) {
|
||||
return ((Boolean) value).toString();
|
||||
} else if (value instanceof Number) {
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
return ExpressionAdaptor.super.toValue(condition, value);
|
||||
}
|
||||
|
||||
public String toLeft(Object left) {
|
||||
if (left instanceof String) {
|
||||
String field = (String) left;
|
||||
if (field.contains(".")) {
|
||||
return field;
|
||||
}
|
||||
return field;
|
||||
}
|
||||
return left.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,794 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.chroma;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.store.condition.ExpressionAdaptor;
|
||||
import com.easyagents.core.model.client.HttpClient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* ChromaVectorStore class provides an interface to interact with Chroma Vector Database
|
||||
* using direct HTTP calls to the Chroma REST API.
|
||||
*/
|
||||
public class ChromaVectorStore extends DocumentStore {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChromaVectorStore.class);
|
||||
private final String baseUrl;
|
||||
private final String collectionName;
|
||||
private final String tenant;
|
||||
private final String database;
|
||||
private final ChromaVectorStoreConfig config;
|
||||
private final ExpressionAdaptor expressionAdaptor;
|
||||
private final HttpClient httpClient;
|
||||
private final int MAX_RETRIES = 3;
|
||||
private final long RETRY_INTERVAL_MS = 1000;
|
||||
|
||||
private static final String BASE_API = "/api/v2";
|
||||
|
||||
public ChromaVectorStore(ChromaVectorStoreConfig config) {
|
||||
Objects.requireNonNull(config, "ChromaVectorStoreConfig cannot be null");
|
||||
this.baseUrl = config.getBaseUrl();
|
||||
this.tenant = config.getTenant();
|
||||
this.database = config.getDatabase();
|
||||
this.collectionName = config.getCollectionName();
|
||||
this.config = config;
|
||||
this.expressionAdaptor = ChromaExpressionAdaptor.DEFAULT;
|
||||
|
||||
// 创建并配置HttpClient实例
|
||||
this.httpClient = createHttpClient();
|
||||
|
||||
// 验证配置的有效性
|
||||
validateConfig();
|
||||
|
||||
// 如果配置了自动创建集合,检查并创建集合
|
||||
if (config.isAutoCreateCollection()) {
|
||||
try {
|
||||
// 确保租户和数据库存在
|
||||
ensureTenantAndDatabaseExists();
|
||||
// 确保集合存在
|
||||
ensureCollectionExists();
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to ensure collection exists: {}. Will retry on first operation.", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private HttpClient createHttpClient() {
|
||||
HttpClient client = new HttpClient();
|
||||
return client;
|
||||
}
|
||||
|
||||
private void validateConfig() {
|
||||
if (baseUrl == null || baseUrl.isEmpty()) {
|
||||
throw new IllegalArgumentException("Base URL cannot be empty");
|
||||
}
|
||||
|
||||
if (!baseUrl.startsWith("http://") && !baseUrl.startsWith("https://")) {
|
||||
throw new IllegalArgumentException("Base URL must start with http:// or https://");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 确保租户和数据库存在,如果不存在则创建
|
||||
*/
|
||||
private void ensureTenantAndDatabaseExists() {
|
||||
try {
|
||||
// 检查并创建租户
|
||||
if (tenant != null && !tenant.isEmpty()) {
|
||||
ensureTenantExists();
|
||||
|
||||
// 检查并创建数据库(如果租户已设置)
|
||||
if (database != null && !database.isEmpty()) {
|
||||
ensureDatabaseExists();
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Error ensuring tenant and database exist", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 确保租户存在,如果不存在则创建
|
||||
*/
|
||||
private void ensureTenantExists() throws IOException {
|
||||
String tenantUrl = baseUrl + BASE_API + "/tenants/" + tenant;
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
try {
|
||||
// 尝试获取租户信息
|
||||
String responseBody = executeWithRetry(() -> httpClient.get(tenantUrl, headers));
|
||||
logger.debug("Successfully verified tenant '{}' exists", tenant);
|
||||
} catch (IOException e) {
|
||||
// 如果获取失败,尝试创建租户
|
||||
logger.info("Creating tenant '{}' as it does not exist", tenant);
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("name", tenant);
|
||||
|
||||
String createTenantUrl = baseUrl + BASE_API + "/tenants";
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(createTenantUrl, headers, jsonRequestBody));
|
||||
logger.info("Successfully created tenant '{}'", tenant);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 确保数据库存在,如果不存在则创建
|
||||
*/
|
||||
private void ensureDatabaseExists() throws IOException {
|
||||
if (tenant == null || tenant.isEmpty()) {
|
||||
throw new IllegalStateException("Cannot create database without tenant");
|
||||
}
|
||||
|
||||
String databaseUrl = baseUrl + BASE_API + "/tenants/" + tenant + "/databases/" + database;
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
try {
|
||||
// 尝试获取数据库信息
|
||||
String responseBody = executeWithRetry(() -> httpClient.get(databaseUrl, headers));
|
||||
logger.debug("Successfully verified database '{}' exists in tenant '{}'",
|
||||
database, tenant);
|
||||
} catch (IOException e) {
|
||||
// 如果获取失败,尝试创建数据库
|
||||
logger.info("Creating database '{}' in tenant '{}' as it does not exist",
|
||||
database, tenant);
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("name", database);
|
||||
|
||||
String createDatabaseUrl = baseUrl + BASE_API + "/tenants/" + tenant + "/databases";
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(createDatabaseUrl, headers, jsonRequestBody));
|
||||
logger.info("Successfully created database '{}' in tenant '{}'",
|
||||
database, tenant);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据collectionName查询Collection ID
|
||||
*/
|
||||
private String getCollectionId(String collectionName) throws IOException {
|
||||
String collectionsUrl = buildCollectionsUrl();
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.get(collectionsUrl, headers));
|
||||
if (responseBody == null) {
|
||||
throw new IOException("Failed to get collections, no response");
|
||||
}
|
||||
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
List<Map<String, Object>> collections = new ArrayList<>();
|
||||
|
||||
// 处理不同格式的响应
|
||||
if (responseObj instanceof Map) {
|
||||
Map<String, Object> responseMap = (Map<String, Object>) responseObj;
|
||||
if (responseMap.containsKey("collections") && responseMap.get("collections") instanceof List) {
|
||||
collections = (List<Map<String, Object>>) responseMap.get("collections");
|
||||
}
|
||||
} else if (responseObj instanceof List) {
|
||||
List<?> rawCollections = (List<?>) responseObj;
|
||||
for (Object item : rawCollections) {
|
||||
if (item instanceof Map) {
|
||||
collections.add((Map<String, Object>) item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 查找指定名称的集合
|
||||
for (Map<String, Object> collection : collections) {
|
||||
if (collection.containsKey("name") && collectionName.equals(collection.get("name"))) {
|
||||
return collection.get("id").toString();
|
||||
}
|
||||
}
|
||||
|
||||
throw new IOException("Collection not found: " + collectionName);
|
||||
}
|
||||
|
||||
private void createCollection() throws IOException {
|
||||
// 构建创建集合的API URL,包含tenant和database
|
||||
String createCollectionUrl = buildCollectionsUrl();
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("name", collectionName);
|
||||
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(createCollectionUrl, headers, jsonRequestBody));
|
||||
if (responseBody == null) {
|
||||
throw new IOException("Failed to create collection: no response");
|
||||
}
|
||||
|
||||
try {
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
|
||||
Map<String, Object> responseMap = null;
|
||||
if (responseObj instanceof Map) {
|
||||
responseMap = (Map<String, Object>) responseObj;
|
||||
}
|
||||
if (responseMap.containsKey("error")) {
|
||||
throw new IOException("Failed to create collection: " + responseMap.get("error"));
|
||||
}
|
||||
|
||||
logger.info("Collection '{}' created successfully", collectionName);
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Failed to process collection creation response: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
Objects.requireNonNull(documents, "Documents cannot be null");
|
||||
|
||||
if (documents.isEmpty()) {
|
||||
logger.debug("No documents to store");
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
try {
|
||||
// 确保集合存在
|
||||
ensureCollectionExists();
|
||||
|
||||
String collectionName = getCollectionName(options);
|
||||
|
||||
List<String> ids = new ArrayList<>();
|
||||
List<List<Double>> embeddings = new ArrayList<>();
|
||||
List<Map<String, Object>> metadatas = new ArrayList<>();
|
||||
List<String> documentsContent = new ArrayList<>();
|
||||
|
||||
for (Document doc : documents) {
|
||||
ids.add(String.valueOf(doc.getId()));
|
||||
|
||||
if (doc.getVector() != null) {
|
||||
List<Double> embedding = doc.getVectorAsDoubleList();
|
||||
embeddings.add(embedding);
|
||||
} else {
|
||||
embeddings.add(null);
|
||||
}
|
||||
|
||||
Map<String, Object> metadata = doc.getMetadataMap() != null ?
|
||||
new HashMap<>(doc.getMetadataMap()) : new HashMap<>();
|
||||
metadatas.add(metadata);
|
||||
|
||||
documentsContent.add(doc.getContent());
|
||||
}
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("ids", ids);
|
||||
requestBody.put("embeddings", embeddings);
|
||||
requestBody.put("metadatas", metadatas);
|
||||
requestBody.put("documents", documentsContent);
|
||||
|
||||
String collectionId = getCollectionId(collectionName);
|
||||
|
||||
// 构建包含tenant和database的完整URL
|
||||
String collectionUrl = buildCollectionUrl(collectionId, "add");
|
||||
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
logger.debug("Storing {} documents to collection '{}'", documents.size(), collectionName);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
|
||||
if (responseBody == null) {
|
||||
logger.error("Error storing documents: no response");
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
|
||||
Map<String, Object> responseMap = null;
|
||||
if (responseObj instanceof Map) {
|
||||
responseMap = (Map<String, Object>) responseObj;
|
||||
}
|
||||
if (responseMap.containsKey("error")) {
|
||||
String errorMsg = "Error storing documents: " + responseMap.get("error");
|
||||
logger.error(errorMsg);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
logger.debug("Successfully stored {} documents", documents.size());
|
||||
return StoreResult.successWithIds(documents);
|
||||
} catch (Exception e) {
|
||||
logger.error("Error storing documents to Chroma", e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
Objects.requireNonNull(ids, "IDs cannot be null");
|
||||
|
||||
if (ids.isEmpty()) {
|
||||
logger.debug("No IDs to delete");
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
try {
|
||||
// 确保集合存在
|
||||
ensureCollectionExists();
|
||||
|
||||
String collectionName = getCollectionName(options);
|
||||
|
||||
List<String> stringIds = ids.stream()
|
||||
.map(Object::toString)
|
||||
.collect(Collectors.toList());
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
requestBody.put("ids", stringIds);
|
||||
|
||||
String collectionId = getCollectionId(collectionName);
|
||||
|
||||
// 构建包含tenant和database的完整URL
|
||||
String collectionUrl = buildCollectionUrl(collectionId, "delete");
|
||||
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
logger.debug("Deleting {} documents from collection '{}'", ids.size(), collectionName);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
|
||||
if (responseBody == null) {
|
||||
logger.error("Error deleting documents: no response");
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
|
||||
Map<String, Object> responseMap = null;
|
||||
if (responseObj instanceof Map) {
|
||||
responseMap = (Map<String, Object>) responseObj;
|
||||
}
|
||||
if (responseMap.containsKey("error")) {
|
||||
String errorMsg = "Error deleting documents: " + responseMap.get("error");
|
||||
logger.error(errorMsg);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
logger.debug("Successfully deleted {} documents", ids.size());
|
||||
return StoreResult.success();
|
||||
} catch (Exception e) {
|
||||
logger.error("Error deleting documents from Chroma", e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
Objects.requireNonNull(documents, "Documents cannot be null");
|
||||
|
||||
if (documents.isEmpty()) {
|
||||
logger.debug("No documents to update");
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
try {
|
||||
// Chroma doesn't support direct update, so we delete and re-add
|
||||
List<Object> ids = documents.stream().map(Document::getId).collect(Collectors.toList());
|
||||
StoreResult deleteResult = doDelete(ids, options);
|
||||
|
||||
if (!deleteResult.isSuccess()) {
|
||||
logger.warn("Delete failed during update operation: {}", deleteResult.toString());
|
||||
// 尝试继续添加,因为可能有些文档是新的
|
||||
}
|
||||
|
||||
StoreResult storeResult = doStore(documents, options);
|
||||
|
||||
if (storeResult.isSuccess()) {
|
||||
logger.debug("Successfully updated {} documents", documents.size());
|
||||
}
|
||||
|
||||
return storeResult;
|
||||
} catch (Exception e) {
|
||||
logger.error("Error updating documents in Chroma", e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
Objects.requireNonNull(wrapper, "SearchWrapper cannot be null");
|
||||
|
||||
try {
|
||||
// 确保集合存在
|
||||
ensureCollectionExists();
|
||||
|
||||
String collectionName = getCollectionName(options);
|
||||
|
||||
int limit = wrapper.getMaxResults() > 0 ? wrapper.getMaxResults() : 10;
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
// 检查查询条件是否有效
|
||||
if (wrapper.getVector() == null && wrapper.getText() == null) {
|
||||
throw new IllegalArgumentException("Either vector or text must be provided for search");
|
||||
}
|
||||
|
||||
// 设置查询向量
|
||||
if (wrapper.getVector() != null) {
|
||||
List<Double> queryEmbedding = wrapper.getVectorAsDoubleList();
|
||||
requestBody.put("query_embeddings", Collections.singletonList(queryEmbedding));
|
||||
logger.debug("Performing vector search with dimension: {}", queryEmbedding.size());
|
||||
} else if (wrapper.getText() != null) {
|
||||
requestBody.put("query_texts", Collections.singletonList(wrapper.getText()));
|
||||
logger.debug("Performing text search: {}", sanitizeLogString(wrapper.getText(), 100));
|
||||
}
|
||||
|
||||
// 设置返回数量
|
||||
requestBody.put("n_results", limit);
|
||||
|
||||
// 设置过滤条件
|
||||
if (wrapper.getCondition() != null) {
|
||||
try {
|
||||
String whereClause = expressionAdaptor.toCondition(wrapper.getCondition());
|
||||
// Chroma的where条件是JSON对象,需要解析
|
||||
Object whereObj = parseJsonResponse(whereClause);
|
||||
|
||||
Map<String, Object> whereMap = null;
|
||||
if (whereObj instanceof Map) {
|
||||
whereMap = (Map<String, Object>) whereObj;
|
||||
}
|
||||
requestBody.put("where", whereMap);
|
||||
logger.debug("Search with filter condition: {}", whereClause);
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to parse filter condition: {}, ignoring condition", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
String collectionId = getCollectionId(collectionName);
|
||||
|
||||
// 构建包含tenant和database的完整URL
|
||||
String collectionUrl = buildCollectionUrl(collectionId, "query");
|
||||
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
|
||||
if (responseBody == null) {
|
||||
logger.error("Error searching documents: no response");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
|
||||
Map<String, Object> responseMap = null;
|
||||
if (responseObj instanceof Map) {
|
||||
responseMap = (Map<String, Object>) responseObj;
|
||||
}
|
||||
|
||||
// 检查响应是否包含error字段
|
||||
if (responseMap.containsKey("error")) {
|
||||
logger.error("Error searching documents: {}", responseMap.get("error"));
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
return parseSearchResults(responseMap);
|
||||
} catch (Exception e) {
|
||||
logger.error("Error searching documents in Chroma", e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 支持直接使用向量数组和topK参数的搜索方法
|
||||
*/
|
||||
public List<Document> searchInternal(double[] vector, int topK, StoreOptions options) {
|
||||
Objects.requireNonNull(vector, "Vector cannot be null");
|
||||
|
||||
if (topK <= 0) {
|
||||
topK = 10;
|
||||
}
|
||||
|
||||
try {
|
||||
// 确保集合存在
|
||||
ensureCollectionExists();
|
||||
|
||||
String collectionName = getCollectionName(options);
|
||||
|
||||
Map<String, Object> requestBody = new HashMap<>();
|
||||
|
||||
// 设置查询向量
|
||||
List<Double> queryEmbedding = Arrays.stream(vector)
|
||||
.boxed()
|
||||
.collect(Collectors.toList());
|
||||
requestBody.put("query_embeddings", Collections.singletonList(queryEmbedding));
|
||||
|
||||
// 设置返回数量
|
||||
requestBody.put("n_results", topK);
|
||||
|
||||
String collectionId = getCollectionId(collectionName);
|
||||
|
||||
// 构建包含tenant和database的完整URL
|
||||
String collectionUrl = buildCollectionUrl(collectionId, "query");
|
||||
|
||||
Map<String, String> headers = createHeaders();
|
||||
|
||||
String jsonRequestBody = safeJsonSerialize(requestBody);
|
||||
|
||||
logger.debug("Performing direct vector search with dimension: {}", vector.length);
|
||||
|
||||
String responseBody = executeWithRetry(() -> httpClient.post(collectionUrl, headers, jsonRequestBody));
|
||||
if (responseBody == null) {
|
||||
logger.error("Error searching documents: no response");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
Object responseObj = parseJsonResponse(responseBody);
|
||||
|
||||
Map<String, Object> responseMap = null;
|
||||
if (responseObj instanceof Map) {
|
||||
responseMap = (Map<String, Object>) responseObj;
|
||||
}
|
||||
|
||||
// 检查响应是否包含error字段
|
||||
if (responseMap.containsKey("error")) {
|
||||
logger.error("Error searching documents: {}", responseMap.get("error"));
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
return parseSearchResults(responseMap);
|
||||
} catch (Exception e) {
|
||||
logger.error("Error searching documents in Chroma", e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
private List<Document> parseSearchResults(Map<String, Object> responseMap) {
|
||||
try {
|
||||
List<String> ids = extractResultsFromNestedList(responseMap, "ids");
|
||||
List<String> documents = extractResultsFromNestedList(responseMap, "documents");
|
||||
List<Map<String, Object>> metadatas = extractResultsFromNestedList(responseMap, "metadatas");
|
||||
List<List<Double>> embeddings = extractResultsFromNestedList(responseMap, "embeddings");
|
||||
List<Double> distances = extractResultsFromNestedList(responseMap, "distances");
|
||||
|
||||
if (ids == null || ids.isEmpty()) {
|
||||
logger.debug("No documents found in search results");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// 转换为Easy-Agents的Document格式
|
||||
List<Document> resultDocs = new ArrayList<>();
|
||||
for (int i = 0; i < ids.size(); i++) {
|
||||
Document doc = new Document();
|
||||
doc.setId(ids.get(i));
|
||||
|
||||
if (documents != null && i < documents.size()) {
|
||||
doc.setContent(documents.get(i));
|
||||
}
|
||||
|
||||
if (metadatas != null && i < metadatas.size()) {
|
||||
doc.setMetadataMap(metadatas.get(i));
|
||||
}
|
||||
|
||||
if (embeddings != null && i < embeddings.size() && embeddings.get(i) != null) {
|
||||
doc.setVector(embeddings.get(i));
|
||||
}
|
||||
|
||||
// 设置相似度分数(距离越小越相似)
|
||||
if (distances != null && i < distances.size()) {
|
||||
double score = 1.0 - distances.get(i);
|
||||
// 确保分数在合理范围内
|
||||
score = Math.max(0, Math.min(1, score));
|
||||
doc.setScore(score);
|
||||
}
|
||||
|
||||
resultDocs.add(doc);
|
||||
}
|
||||
|
||||
logger.debug("Found {} documents in search results", resultDocs.size());
|
||||
return resultDocs;
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to parse search results", e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private <T> List<T> extractResultsFromNestedList(Map<String, Object> responseMap, String key) {
|
||||
try {
|
||||
if (!responseMap.containsKey(key)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<?> outerList = (List<?>) responseMap.get(key);
|
||||
if (outerList == null || outerList.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Chroma返回的结果是嵌套列表,第一个元素是当前查询的结果
|
||||
return (List<T>) outerList.get(0);
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to extract '{}' from response: {}", key, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> createHeaders() {
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
|
||||
if (config.getApiKey() != null && !config.getApiKey().isEmpty()) {
|
||||
headers.put("X-Chroma-Token", config.getApiKey());
|
||||
}
|
||||
|
||||
// 添加租户和数据库信息(如果配置了)
|
||||
if (tenant != null && !tenant.isEmpty()) {
|
||||
headers.put("X-Chroma-Tenant", tenant);
|
||||
}
|
||||
|
||||
if (database != null && !database.isEmpty()) {
|
||||
headers.put("X-Chroma-Database", database);
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private <T> T executeWithRetry(HttpOperation<T> operation) throws IOException {
|
||||
int attempts = 0;
|
||||
IOException lastException = null;
|
||||
|
||||
while (attempts < MAX_RETRIES) {
|
||||
try {
|
||||
attempts++;
|
||||
return operation.execute();
|
||||
} catch (IOException e) {
|
||||
lastException = e;
|
||||
|
||||
// 如果是最后一次尝试,则抛出异常
|
||||
if (attempts >= MAX_RETRIES) {
|
||||
throw new IOException("Operation failed after " + MAX_RETRIES + " attempts: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
// 记录重试信息
|
||||
logger.warn("Operation failed (attempt {} of {}), retrying in {}ms: {}",
|
||||
attempts, MAX_RETRIES, RETRY_INTERVAL_MS, e.getMessage());
|
||||
|
||||
// 等待一段时间后重试
|
||||
try {
|
||||
Thread.sleep(RETRY_INTERVAL_MS);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("Retry interrupted", ie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 这一行理论上不会执行到,但为了编译器满意
|
||||
throw lastException != null ? lastException : new IOException("Operation failed without exception");
|
||||
}
|
||||
|
||||
private String safeJsonSerialize(Map<String, Object> map) {
|
||||
// 使用标准的JSON序列化,但在实际应用中可以添加更多的安全检查
|
||||
try {
|
||||
return new com.google.gson.Gson().toJson(map);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to serialize request body to JSON", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Object parseJsonResponse(String json) {
|
||||
try {
|
||||
if (json == null || json.trim().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
// Check if JSON starts with [ indicating an array
|
||||
if (json.trim().startsWith("[")) {
|
||||
return new com.google.gson.Gson().fromJson(json, List.class);
|
||||
} else {
|
||||
// Otherwise assume it's an object
|
||||
return new com.google.gson.Gson().fromJson(json, Map.class);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to parse JSON response: " + json, e);
|
||||
}
|
||||
}
|
||||
|
||||
private String sanitizeLogString(String input, int maxLength) {
|
||||
if (input == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String sanitized = input.replaceAll("[\n\r]", " ");
|
||||
return sanitized.length() > maxLength ? sanitized.substring(0, maxLength) + "..." : sanitized;
|
||||
}
|
||||
|
||||
private String getCollectionName(StoreOptions options) {
|
||||
return options != null ? options.getCollectionNameOrDefault(collectionName) : collectionName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建特定集合操作的URL,包含tenant和database
|
||||
*/
|
||||
private String buildCollectionUrl(String collectionId, String operation) {
|
||||
StringBuilder urlBuilder = new StringBuilder(baseUrl).append(BASE_API);
|
||||
|
||||
if (tenant != null && !tenant.isEmpty()) {
|
||||
urlBuilder.append("/tenants/").append(tenant);
|
||||
|
||||
if (database != null && !database.isEmpty()) {
|
||||
urlBuilder.append("/databases/").append(database);
|
||||
}
|
||||
}
|
||||
|
||||
urlBuilder.append("/collections/").append(collectionId).append("/").append(operation);
|
||||
return urlBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the connection to Chroma database
|
||||
*/
|
||||
public void close() {
|
||||
// HttpClient类使用连接池管理,这里可以添加额外的资源清理逻辑
|
||||
logger.info("Chroma client closed");
|
||||
}
|
||||
|
||||
/**
|
||||
* 确保集合存在,如果不存在则创建
|
||||
*/
|
||||
private void ensureCollectionExists() throws IOException {
|
||||
try {
|
||||
// 尝试获取默认集合ID,如果能获取到则说明集合存在
|
||||
getCollectionId(collectionName);
|
||||
logger.debug("Collection '{}' exists", collectionName);
|
||||
} catch (IOException e) {
|
||||
// 如果获取集合ID失败,说明集合不存在,需要创建
|
||||
logger.info("Collection '{}' does not exist, creating...", collectionName);
|
||||
createCollection();
|
||||
logger.info("Collection '{}' created successfully", collectionName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建集合列表URL,包含tenant和database
|
||||
*/
|
||||
private String buildCollectionsUrl() {
|
||||
StringBuilder urlBuilder = new StringBuilder(baseUrl).append(BASE_API);
|
||||
|
||||
if (tenant != null && !tenant.isEmpty()) {
|
||||
urlBuilder.append("/tenants/").append(tenant);
|
||||
|
||||
if (database != null && !database.isEmpty()) {
|
||||
urlBuilder.append("/databases/").append(database);
|
||||
}
|
||||
}
|
||||
|
||||
urlBuilder.append("/collections");
|
||||
return urlBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 函数式接口,用于封装HTTP操作以支持重试
|
||||
*/
|
||||
private interface HttpOperation<T> {
|
||||
T execute() throws IOException;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.chroma;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* ChromaVectorStoreConfig class provides configuration for ChromaVectorStore.
|
||||
*/
|
||||
public class ChromaVectorStoreConfig implements DocumentStoreConfig {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChromaVectorStoreConfig.class);
|
||||
|
||||
private String host = "localhost";
|
||||
private int port = 8000;
|
||||
private String collectionName;
|
||||
private boolean autoCreateCollection = true;
|
||||
private String apiKey;
|
||||
private String tenant;
|
||||
private String database;
|
||||
|
||||
public ChromaVectorStoreConfig() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the host of Chroma database
|
||||
*
|
||||
* @return the host of Chroma database
|
||||
*/
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the host of Chroma database
|
||||
*
|
||||
* @param host the host of Chroma database
|
||||
*/
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the port of Chroma database
|
||||
*
|
||||
* @return the port of Chroma database
|
||||
*/
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the port of Chroma database
|
||||
*
|
||||
* @param port the port of Chroma database
|
||||
*/
|
||||
public void setPort(int port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the collection name of Chroma database
|
||||
*
|
||||
* @return the collection name of Chroma database
|
||||
*/
|
||||
public String getCollectionName() {
|
||||
return collectionName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the collection name of Chroma database
|
||||
*
|
||||
* @param collectionName the collection name of Chroma database
|
||||
*/
|
||||
public void setCollectionName(String collectionName) {
|
||||
this.collectionName = collectionName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get whether to automatically create the collection if it doesn't exist
|
||||
*
|
||||
* @return true if the collection should be created automatically, false otherwise
|
||||
*/
|
||||
public boolean isAutoCreateCollection() {
|
||||
return autoCreateCollection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to automatically create the collection if it doesn't exist
|
||||
*
|
||||
* @param autoCreateCollection true if the collection should be created automatically, false otherwise
|
||||
*/
|
||||
public void setAutoCreateCollection(boolean autoCreateCollection) {
|
||||
this.autoCreateCollection = autoCreateCollection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the API key of Chroma database
|
||||
*
|
||||
* @return the API key of Chroma database
|
||||
*/
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the API key of Chroma database
|
||||
*
|
||||
* @param apiKey the API key of Chroma database
|
||||
*/
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the tenant of Chroma database
|
||||
*
|
||||
* @return the tenant of Chroma database
|
||||
*/
|
||||
public String getTenant() {
|
||||
return tenant;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the tenant of Chroma database
|
||||
*
|
||||
* @param tenant the tenant of Chroma database
|
||||
*/
|
||||
public void setTenant(String tenant) {
|
||||
this.tenant = tenant;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the database of Chroma database
|
||||
*
|
||||
* @return the database of Chroma database
|
||||
*/
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the database of Chroma database
|
||||
*
|
||||
* @param database the database of Chroma database
|
||||
*/
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
try {
|
||||
URL url = new URL(getBaseUrl() + "/api/v2/heartbeat");
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||
connection.setRequestMethod("GET");
|
||||
connection.setConnectTimeout(5000);
|
||||
connection.setReadTimeout(5000);
|
||||
|
||||
if (apiKey != null && !apiKey.isEmpty()) {
|
||||
connection.setRequestProperty("X-Chroma-Token", apiKey);
|
||||
}
|
||||
|
||||
int responseCode = connection.getResponseCode();
|
||||
connection.disconnect();
|
||||
|
||||
return responseCode == 200;
|
||||
} catch (IOException e) {
|
||||
logger.warn("Chroma database is not available: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the base URL of Chroma database
|
||||
*
|
||||
* @return the base URL of Chroma database
|
||||
*/
|
||||
public String getBaseUrl() {
|
||||
return "http://" + host + ":" + port;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.chroma;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.model.client.HttpClient;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assume;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
* ChromaVectorStore的测试类,测试文档的存储、搜索、更新和删除功能
|
||||
* 包含连接检查和错误处理机制,支持在无真实Chroma服务器时跳过测试
|
||||
*/
|
||||
public class ChromaVectorStoreTest {
|
||||
|
||||
private static ChromaVectorStore store;
|
||||
private static String testTenant = "default_tenant";
|
||||
private static String testDatabase = "default_database";
|
||||
private static String testCollectionName = "test_collection";
|
||||
private static boolean isChromaAvailable = false;
|
||||
private static boolean useMock = false; // 设置为true可以在没有真实Chroma服务器时使用模拟模式
|
||||
|
||||
/**
|
||||
* 在测试开始前初始化ChromaVectorStore实例
|
||||
*/
|
||||
@BeforeClass
|
||||
public static void setUp() {
|
||||
// 创建配置对象
|
||||
ChromaVectorStoreConfig config = new ChromaVectorStoreConfig();
|
||||
config.setHost("localhost");
|
||||
config.setPort(8000);
|
||||
config.setCollectionName(testCollectionName);
|
||||
config.setTenant(testTenant);
|
||||
config.setDatabase(testDatabase);
|
||||
config.setAutoCreateCollection(true);
|
||||
|
||||
// 初始化存储实例
|
||||
try {
|
||||
store = new ChromaVectorStore(config);
|
||||
System.out.println("ChromaVectorStore initialized successfully.");
|
||||
|
||||
// 检查连接是否可用
|
||||
isChromaAvailable = checkChromaConnection(config);
|
||||
|
||||
if (!isChromaAvailable && !useMock) {
|
||||
System.out.println("Chroma server is not available. Tests will be skipped unless useMock is set to true.");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to initialize ChromaVectorStore: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查Chroma服务器连接是否可用
|
||||
*/
|
||||
private static boolean checkChromaConnection(ChromaVectorStoreConfig config) {
|
||||
try {
|
||||
String baseUrl = "http://" + config.getHost() + ":" + config.getPort();
|
||||
String healthCheckUrl = baseUrl + "/api/v2/heartbeat";
|
||||
|
||||
HttpClient httpClient = new HttpClient();
|
||||
System.out.println("Checking Chroma server connection at: " + healthCheckUrl);
|
||||
|
||||
// 使用较短的超时时间进行健康检查
|
||||
String response = httpClient.get(healthCheckUrl);
|
||||
if (response != null) {
|
||||
System.out.println("Chroma server connection successful! Response: " + response);
|
||||
return true;
|
||||
} else {
|
||||
System.out.println("Chroma server connection failed: Empty response");
|
||||
return false;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("Chroma server connection failed: " + e.getMessage());
|
||||
System.out.println("Please ensure Chroma server is running on http://" + config.getHost() + ":" + config.getPort());
|
||||
System.out.println("To run tests without a real Chroma server, set 'useMock = true'");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查是否应该运行测试
|
||||
*/
|
||||
private void assumeChromaAvailable() {
|
||||
Assume.assumeTrue("Chroma server is not available and mock mode is disabled",
|
||||
isChromaAvailable || useMock);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在所有测试完成后清理资源
|
||||
*/
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
if (store != null) {
|
||||
try {
|
||||
store.close();
|
||||
System.out.println("ChromaVectorStore closed successfully.");
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error closing ChromaVectorStore: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试存储文档功能
|
||||
*/
|
||||
@Test
|
||||
public void testStoreDocuments() {
|
||||
assumeChromaAvailable();
|
||||
|
||||
System.out.println("Starting testStoreDocuments...");
|
||||
|
||||
// 创建测试文档
|
||||
List<Document> documents = createTestDocuments();
|
||||
|
||||
// 如果使用模拟模式,直接返回成功结果
|
||||
if (useMock) {
|
||||
System.out.println("Running in mock mode. Simulating store operation.");
|
||||
StoreResult mockResult = StoreResult.successWithIds(documents);
|
||||
assertTrue("Mock store operation should be successful", mockResult.isSuccess());
|
||||
assertEquals("All document IDs should be returned in mock mode",
|
||||
documents.size(), mockResult.ids().size());
|
||||
System.out.println("testStoreDocuments completed successfully in mock mode.");
|
||||
return;
|
||||
}
|
||||
|
||||
// 存储文档
|
||||
try {
|
||||
StoreResult result = store.doStore(documents, StoreOptions.DEFAULT);
|
||||
System.out.println("Store result: " + result);
|
||||
|
||||
// 验证存储是否成功
|
||||
assertTrue("Store operation should be successful", result.isSuccess());
|
||||
assertEquals("All document IDs should be returned", documents.size(), result.ids().size());
|
||||
|
||||
System.out.println("testStoreDocuments completed successfully.");
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to store documents: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
fail("Store operation failed with exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试搜索文档功能
|
||||
*/
|
||||
@Test
|
||||
public void testSearchDocuments() {
|
||||
assumeChromaAvailable();
|
||||
|
||||
System.out.println("Starting testSearchDocuments...");
|
||||
|
||||
// 创建测试文档
|
||||
List<Document> documents = createTestDocuments();
|
||||
|
||||
// 如果使用模拟模式
|
||||
if (useMock) {
|
||||
System.out.println("Running in mock mode. Simulating search operation.");
|
||||
// 模拟搜索结果,返回前3个文档
|
||||
List<Document> mockResults = new ArrayList<>(documents.subList(0, Math.min(3, documents.size())));
|
||||
for (int i = 0; i < mockResults.size(); i++) {
|
||||
mockResults.get(i).setScore(1.0 - i * 0.1); // 模拟相似度分数
|
||||
}
|
||||
|
||||
// 验证模拟结果
|
||||
assertNotNull("Mock search results should not be null", mockResults);
|
||||
assertFalse("Mock search results should not be empty", mockResults.isEmpty());
|
||||
assertTrue("Mock search results should have the correct maximum size", mockResults.size() <= 3);
|
||||
|
||||
System.out.println("testSearchDocuments completed successfully in mock mode.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 首先存储一些测试文档
|
||||
store.doStore(documents, StoreOptions.DEFAULT);
|
||||
|
||||
// 创建搜索包装器
|
||||
SearchWrapper searchWrapper = new SearchWrapper();
|
||||
// 使用第一个文档的向量进行搜索
|
||||
searchWrapper.setVector(documents.get(0).getVector());
|
||||
searchWrapper.setMaxResults(3);
|
||||
|
||||
// 执行搜索
|
||||
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
|
||||
|
||||
// 验证搜索结果
|
||||
assertNotNull("Search results should not be null", searchResults);
|
||||
assertFalse("Search results should not be empty", searchResults.isEmpty());
|
||||
assertTrue("Search results should have the correct maximum size",
|
||||
searchResults.size() <= searchWrapper.getMaxResults());
|
||||
|
||||
// 打印搜索结果
|
||||
System.out.println("Search results:");
|
||||
for (Document doc : searchResults) {
|
||||
System.out.printf("id=%s, content=%s, vector=%s, score=%s\n",
|
||||
doc.getId(), doc.getContent(), Arrays.toString(doc.getVector()), doc.getScore());
|
||||
}
|
||||
|
||||
System.out.println("testSearchDocuments completed successfully.");
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to search documents: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
fail("Search operation failed with exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试更新文档功能
|
||||
*/
|
||||
@Test
|
||||
public void testUpdateDocuments() {
|
||||
assumeChromaAvailable();
|
||||
|
||||
System.out.println("Starting testUpdateDocuments...");
|
||||
|
||||
// 创建测试文档
|
||||
List<Document> documents = createTestDocuments();
|
||||
|
||||
// 如果使用模拟模式
|
||||
if (useMock) {
|
||||
System.out.println("Running in mock mode. Simulating update operation.");
|
||||
|
||||
// 修改文档内容
|
||||
Document updatedDoc = documents.get(0);
|
||||
String originalContent = updatedDoc.getContent();
|
||||
updatedDoc.setContent(originalContent + " [UPDATED]");
|
||||
|
||||
// 模拟更新结果
|
||||
StoreResult mockResult = StoreResult.successWithIds(Arrays.asList(updatedDoc));
|
||||
assertTrue("Mock update operation should be successful", mockResult.isSuccess());
|
||||
|
||||
System.out.println("testUpdateDocuments completed successfully in mock mode.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 首先存储一些测试文档
|
||||
store.doStore(documents, StoreOptions.DEFAULT);
|
||||
|
||||
// 修改文档内容
|
||||
Document updatedDoc = documents.get(0);
|
||||
String originalContent = updatedDoc.getContent();
|
||||
updatedDoc.setContent(originalContent + " [UPDATED]");
|
||||
|
||||
// 执行更新
|
||||
StoreResult result = store.doUpdate(Arrays.asList(updatedDoc), StoreOptions.DEFAULT);
|
||||
|
||||
// 验证更新是否成功
|
||||
assertTrue("Update operation should be successful", result.isSuccess());
|
||||
|
||||
// 搜索更新后的文档以验证更改
|
||||
SearchWrapper searchWrapper = new SearchWrapper();
|
||||
searchWrapper.setVector(updatedDoc.getVector());
|
||||
searchWrapper.setMaxResults(1);
|
||||
|
||||
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
|
||||
assertTrue("Should find the updated document", !searchResults.isEmpty());
|
||||
assertEquals("Document content should be updated",
|
||||
updatedDoc.getContent(), searchResults.get(0).getContent());
|
||||
|
||||
System.out.println("testUpdateDocuments completed successfully.");
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to update documents: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
fail("Update operation failed with exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试删除文档功能
|
||||
*/
|
||||
@Test
|
||||
public void testDeleteDocuments() {
|
||||
assumeChromaAvailable();
|
||||
|
||||
System.out.println("Starting testDeleteDocuments...");
|
||||
|
||||
// 创建测试文档
|
||||
List<Document> documents = createTestDocuments();
|
||||
|
||||
// 如果使用模拟模式
|
||||
if (useMock) {
|
||||
System.out.println("Running in mock mode. Simulating delete operation.");
|
||||
|
||||
// 获取要删除的文档ID
|
||||
List<Object> idsToDelete = new ArrayList<>();
|
||||
idsToDelete.add(documents.get(0).getId());
|
||||
|
||||
// 模拟删除结果
|
||||
StoreResult mockResult = StoreResult.success();
|
||||
assertTrue("Mock delete operation should be successful", mockResult.isSuccess());
|
||||
|
||||
System.out.println("testDeleteDocuments completed successfully in mock mode.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 首先存储一些测试文档
|
||||
store.doStore(documents, StoreOptions.DEFAULT);
|
||||
|
||||
// 获取要删除的文档ID
|
||||
List<Object> idsToDelete = new ArrayList<>();
|
||||
idsToDelete.add(documents.get(0).getId());
|
||||
|
||||
// 执行删除
|
||||
StoreResult result = store.doDelete(idsToDelete, StoreOptions.DEFAULT);
|
||||
|
||||
// 验证删除是否成功
|
||||
assertTrue("Delete operation should be successful", result.isSuccess());
|
||||
|
||||
// 尝试搜索已删除的文档
|
||||
SearchWrapper searchWrapper = new SearchWrapper();
|
||||
searchWrapper.setVector(documents.get(0).getVector());
|
||||
searchWrapper.setMaxResults(10);
|
||||
|
||||
List<Document> searchResults = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
|
||||
|
||||
// 检查结果中是否包含已删除的文档
|
||||
boolean deletedDocFound = searchResults.stream()
|
||||
.anyMatch(doc -> doc.getId().equals(documents.get(0).getId()));
|
||||
|
||||
assertFalse("Deleted document should not be found", deletedDocFound);
|
||||
|
||||
System.out.println("testDeleteDocuments completed successfully.");
|
||||
} catch (Exception e) {
|
||||
System.err.println("Failed to delete documents: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
fail("Delete operation failed with exception: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建测试文档
|
||||
*/
|
||||
private List<Document> createTestDocuments() {
|
||||
List<Document> documents = new ArrayList<>();
|
||||
|
||||
// 创建5个测试文档,每个文档都有不同的内容和向量
|
||||
for (int i = 0; i < 5; i++) {
|
||||
Document doc = new Document();
|
||||
doc.setId("doc_" + i);
|
||||
doc.setContent("This is test document content " + i);
|
||||
doc.setTitle("Test Document " + i);
|
||||
|
||||
// 创建一个简单的向量,向量维度为10
|
||||
float[] vector = new float[10];
|
||||
for (int j = 0; j < vector.length; j++) {
|
||||
vector[j] = i + j * 0.1f;
|
||||
}
|
||||
doc.setVector(vector);
|
||||
|
||||
documents.add(doc);
|
||||
}
|
||||
|
||||
return documents;
|
||||
}
|
||||
}
|
||||
49
easy-agents-store/easy-agents-store-elasticsearch/pom.xml
Normal file
49
easy-agents-store/easy-agents-store-elasticsearch/pom.xml
Normal file
@@ -0,0 +1,49 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-elasticsearch</name>
|
||||
<artifactId>easy-agents-store-elasticsearch</artifactId>
|
||||
|
||||
<properties>
|
||||
<elasticsearch.version>8.15.0</elasticsearch.version>
|
||||
<jackson.version>2.17.0</jackson.version>
|
||||
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>co.elastic.clients</groupId>
|
||||
<artifactId>elasticsearch-java</artifactId>
|
||||
<version>${elasticsearch.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,282 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.elasticsearch;
|
||||
|
||||
import co.elastic.clients.elasticsearch.ElasticsearchClient;
|
||||
import co.elastic.clients.elasticsearch._types.ErrorCause;
|
||||
import co.elastic.clients.elasticsearch._types.mapping.DenseVectorProperty;
|
||||
import co.elastic.clients.elasticsearch._types.mapping.Property;
|
||||
import co.elastic.clients.elasticsearch._types.mapping.TextProperty;
|
||||
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
|
||||
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
|
||||
import co.elastic.clients.elasticsearch._types.query_dsl.ScriptScoreQuery;
|
||||
import co.elastic.clients.elasticsearch.core.BulkRequest;
|
||||
import co.elastic.clients.elasticsearch.core.BulkResponse;
|
||||
import co.elastic.clients.elasticsearch.core.SearchRequest;
|
||||
import co.elastic.clients.elasticsearch.core.SearchResponse;
|
||||
import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem;
|
||||
import co.elastic.clients.json.JsonData;
|
||||
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
|
||||
import co.elastic.clients.transport.ElasticsearchTransport;
|
||||
import co.elastic.clients.transport.endpoints.BooleanResponse;
|
||||
import co.elastic.clients.transport.rest_client.RestClientTransport;
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.store.exception.StoreException;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.apache.http.ssl.SSLContextBuilder;
|
||||
import org.elasticsearch.client.RestClient;
|
||||
import org.elasticsearch.client.RestClientBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import java.io.IOException;
|
||||
import java.security.KeyManagementException;
|
||||
import java.security.KeyStoreException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* es 向量存储:<a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/introduction.html">elasticsearch-java</a>
|
||||
*
|
||||
* @author songyinyin
|
||||
* @since 2024/8/12 下午4:17
|
||||
*/
|
||||
public class ElasticSearchVectorStore extends DocumentStore {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ElasticSearchVectorStore.class);
|
||||
|
||||
private final ElasticsearchClient client;
|
||||
|
||||
private final ElasticSearchVectorStoreConfig config;
|
||||
|
||||
public ElasticSearchVectorStore(ElasticSearchVectorStoreConfig config) {
|
||||
this.config = config;
|
||||
RestClientBuilder restClientBuilder = RestClient.builder(HttpHost.create(config.getServerUrl()));
|
||||
|
||||
try {
|
||||
SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build();
|
||||
|
||||
if (StringUtil.hasText(config.getUsername())) {
|
||||
CredentialsProvider provider = new BasicCredentialsProvider();
|
||||
provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword()));
|
||||
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
|
||||
httpClientBuilder.setSSLContext(sslContext);
|
||||
httpClientBuilder.setDefaultCredentialsProvider(provider);
|
||||
return httpClientBuilder;
|
||||
});
|
||||
}
|
||||
|
||||
if (StringUtil.hasText(config.getApiKey())) {
|
||||
restClientBuilder.setDefaultHeaders(new Header[]{
|
||||
new BasicHeader("Authorization", "Apikey " + config.getApiKey())
|
||||
});
|
||||
}
|
||||
|
||||
ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper());
|
||||
|
||||
this.client = new ElasticsearchClient(transport);
|
||||
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
|
||||
throw new StoreException("Elasticsearch init error", e);
|
||||
}
|
||||
try {
|
||||
client.ping();
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O Elasticsearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public ElasticSearchVectorStore(ElasticSearchVectorStoreConfig config, ElasticsearchClient client) {
|
||||
this.config = config;
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
private static void throwIfError(BulkResponse bulkResponse) {
|
||||
if (bulkResponse.errors()) {
|
||||
for (BulkResponseItem item : bulkResponse.items()) {
|
||||
if (item.error() == null) {
|
||||
continue;
|
||||
}
|
||||
ErrorCause errorCause = item.error();
|
||||
throw new StoreException("type: " + errorCause.type() + "," + "reason: " + errorCause.reason());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
String indexName;
|
||||
if (StringUtil.hasText(options.getCollectionName())){
|
||||
indexName = options.getCollectionName();
|
||||
} else {
|
||||
indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
}
|
||||
createIndexIfNotExist(indexName);
|
||||
return saveOrUpdate(documents, indexName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
|
||||
for (Object id : ids) {
|
||||
bulkBuilder.operations(op -> op.delete(d -> d.index(indexName).id(id.toString())));
|
||||
}
|
||||
bulk(bulkBuilder.build());
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
return saveOrUpdate(documents, indexName);
|
||||
}
|
||||
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
// 最小匹配分数,无值则默认0
|
||||
Double minScore = wrapper.getMinScore();
|
||||
// 获取索引名,无指定则使用配置的默认索引
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
|
||||
// 公式:(cosineSimilarity + 1.0) / 2 将相似度映射到 0~1 区间
|
||||
ScriptScoreQuery scriptScoreQuery = ScriptScoreQuery.of(fn -> fn
|
||||
.minScore(minScore == null ? 0 : minScore.floatValue())
|
||||
.query(Query.of(q -> q.matchAll(m -> m)))
|
||||
.script(s -> s
|
||||
.source("(cosineSimilarity(params.query_vector, 'vector') + 1.0) / 2")
|
||||
.params("query_vector", JsonData.of(wrapper.getVector()))
|
||||
)
|
||||
);
|
||||
|
||||
try {
|
||||
SearchResponse<JsonData> response = client.search(
|
||||
SearchRequest.of(s -> s.index(indexName)
|
||||
.query(n -> n.scriptScore(scriptScoreQuery))
|
||||
.size(wrapper.getMaxResults())),
|
||||
JsonData.class
|
||||
);
|
||||
|
||||
return response.hits().hits().stream()
|
||||
.filter(hit -> hit.source() != null) // 过滤_source为空的无效结果
|
||||
.map(hit -> parseFromJsonData(hit.source(), hit.score()))
|
||||
.collect(Collectors.toList());
|
||||
} catch (IOException e) {
|
||||
log.error("[es/search] Elasticsearch I/O exception occurred", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private StoreResult saveOrUpdate(List<Document> documents, String indexName) {
|
||||
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
|
||||
for (Document document : documents) {
|
||||
bulkBuilder.operations(op -> op.index(
|
||||
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
|
||||
);
|
||||
}
|
||||
bulk(bulkBuilder.build());
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
private void bulk(BulkRequest bulkRequest) {
|
||||
try {
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest);
|
||||
throwIfError(bulkResponse);
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O Elasticsearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void createIndexIfNotExist(String indexName) {
|
||||
try {
|
||||
BooleanResponse response = client.indices().exists(c -> c.index(indexName));
|
||||
if (!response.value()) {
|
||||
log.info("[ElasticSearch] Index {} not exists, creating...", indexName);
|
||||
client.indices().create(c -> c.index(indexName)
|
||||
.mappings(getDefaultMappings(this.getEmbeddingModel().dimensions())));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O ElasticSearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private TypeMapping getDefaultMappings(int dimension) {
|
||||
Map<String, Property> properties = new HashMap<>(4);
|
||||
properties.put("content", Property.of(p -> p.text(TextProperty.of(t -> t))));
|
||||
properties.put("vector", Property.of(p -> p.denseVector(DenseVectorProperty.of(d -> d.dims(dimension)))));
|
||||
return TypeMapping.of(c -> c.properties(properties));
|
||||
}
|
||||
|
||||
private Document parseFromJsonData(JsonData source, Double score) {
|
||||
Document document = new Document();
|
||||
Map<String, Object> dataMap = source.to(Map.class);
|
||||
|
||||
document.setId(dataMap.get("id"));
|
||||
document.setTitle((String) dataMap.get("title"));
|
||||
document.setContent((String) dataMap.get("content"));
|
||||
document.setScore(score);
|
||||
|
||||
Object vectorObj = dataMap.get("vector");
|
||||
if (vectorObj instanceof List<?>) {
|
||||
List<?> vectorList = (List<?>) vectorObj;
|
||||
float[] vector = new float[vectorList.size()];
|
||||
for (int i = 0; i < vectorList.size(); i++) {
|
||||
Object val = vectorList.get(i);
|
||||
if (val instanceof Number) {
|
||||
vector[i] = ((Number) val).floatValue();
|
||||
}
|
||||
}
|
||||
document.setVector(vector);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> metadataMap = (Map<String, Object>) dataMap.get("metadataMap");
|
||||
if (metadataMap != null && !metadataMap.isEmpty()) {
|
||||
document.setMetadataMap(metadataMap);
|
||||
} else {
|
||||
Map<String, Object> otherMetadata = new HashMap<>();
|
||||
for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if (!"id".equals(key) && !"title".equals(key)
|
||||
&& !"content".equals(key) && !"vector".equals(key)) {
|
||||
otherMetadata.put(key, entry.getValue());
|
||||
}
|
||||
}
|
||||
if (!otherMetadata.isEmpty()) {
|
||||
document.setMetadataMap(otherMetadata);
|
||||
}
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.elasticsearch;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
/**
|
||||
* 连接 elasticsearch 配置:<a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/getting-started-java.html">elasticsearch-java</a>
|
||||
*
|
||||
* @author songyinyin
|
||||
*/
|
||||
public class ElasticSearchVectorStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String serverUrl = "https://localhost:9200";
|
||||
|
||||
private String apiKey;
|
||||
|
||||
private String username;
|
||||
|
||||
private String password;
|
||||
|
||||
private String defaultIndexName = "easy-agents-default";
|
||||
|
||||
public String getServerUrl() {
|
||||
return serverUrl;
|
||||
}
|
||||
|
||||
public void setServerUrl(String serverUrl) {
|
||||
this.serverUrl = serverUrl;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getDefaultIndexName() {
|
||||
return defaultIndexName;
|
||||
}
|
||||
|
||||
public void setDefaultIndexName(String defaultIndexName) {
|
||||
this.defaultIndexName = defaultIndexName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.serverUrl, this.apiKey, this.defaultIndexName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.opensearch;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||
import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.VectorData;
|
||||
import com.easyagents.core.store.exception.StoreException;
|
||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
|
||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author songyinyin
|
||||
*/
|
||||
public class ElasticSearchVectorStoreTest {
|
||||
|
||||
private static ElasticSearchVectorStore getVectorStore() {
|
||||
ElasticSearchVectorStoreConfig config = new ElasticSearchVectorStoreConfig();
|
||||
// config.setApiKey("bmtXRVNaRUJNMEZXZzMzcnNvSXk6MlNMVmFnT0hRVVNUSmN3UXpoNWp4Zw==");
|
||||
config.setUsername("elastic");
|
||||
config.setPassword("Dd2024a10");
|
||||
ElasticSearchVectorStore store = new ElasticSearchVectorStore(config);
|
||||
store.setEmbeddingModel(new EmbeddingModel() {
|
||||
@Override
|
||||
public VectorData embed(Document document, EmbeddingOptions options) {
|
||||
VectorData vectorData = new VectorData();
|
||||
vectorData.setVector(new float[]{0, 0});
|
||||
return vectorData;
|
||||
}
|
||||
});
|
||||
return store;
|
||||
}
|
||||
|
||||
@Test(expected = StoreException.class)
|
||||
public void test01() {
|
||||
ElasticSearchVectorStore store = getVectorStore();
|
||||
|
||||
// https://opensearch.org/docs/latest/search-plugins/vector-search/#example
|
||||
List<Document> list = new ArrayList<>();
|
||||
Document doc1 = new Document();
|
||||
doc1.setId(1);
|
||||
doc1.setContent("test1");
|
||||
doc1.setVector(new float[]{5.2f, 4.4f});
|
||||
list.add(doc1);
|
||||
Document doc2 = new Document();
|
||||
doc2.setId(2);
|
||||
doc2.setContent("test2");
|
||||
doc2.setVector(new float[]{5.2f, 3.9f});
|
||||
list.add(doc2);
|
||||
Document doc3 = new Document();
|
||||
doc3.setId(3);
|
||||
doc3.setContent("test3");
|
||||
doc3.setVector(new float[]{4.9f, 3.4f});
|
||||
list.add(doc3);
|
||||
Document doc4 = new Document();
|
||||
doc4.setId(4);
|
||||
doc4.setContent("test4");
|
||||
doc4.setVector(new float[]{4.2f, 4.6f});
|
||||
list.add(doc4);
|
||||
Document doc5 = new Document();
|
||||
doc5.setId(5);
|
||||
doc5.setContent("test5");
|
||||
doc5.setVector(new float[]{3.3f, 4.5f});
|
||||
list.add(doc5);
|
||||
store.doStore(list, StoreOptions.DEFAULT);
|
||||
|
||||
// 可能要等一会 才能查出结果
|
||||
SearchWrapper searchWrapper = new SearchWrapper();
|
||||
searchWrapper.setVector(new float[]{5, 4});
|
||||
searchWrapper.setMaxResults(3);
|
||||
List<Document> documents = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
|
||||
for (Document document : documents) {
|
||||
System.out.printf("id=%s, content=%s, vector=%s, metadata=%s\n",
|
||||
document.getId(), document.getContent(), Arrays.toString(document.getVector()), document.getMetadataMap());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
56
easy-agents-store/easy-agents-store-opensearch/pom.xml
Normal file
56
easy-agents-store/easy-agents-store-opensearch/pom.xml
Normal file
@@ -0,0 +1,56 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-opensearch</name>
|
||||
<artifactId>easy-agents-store-opensearch</artifactId>
|
||||
|
||||
<properties>
|
||||
<opensearch.version>2.13.0</opensearch.version>
|
||||
<httpclient5.version>5.1.4</httpclient5.version>
|
||||
<httpcore5-h2>5.1.4</httpcore5-h2>
|
||||
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.opensearch.client</groupId>
|
||||
<artifactId>opensearch-java</artifactId>
|
||||
<version>${opensearch.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents.client5</groupId>
|
||||
<artifactId>httpclient5</artifactId>
|
||||
<version>${httpclient5.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents.core5</groupId>
|
||||
<artifactId>httpcore5-h2</artifactId>
|
||||
<version>${httpcore5-h2}</version> <!-- 版本需与 httpclient5 一致 -->
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,264 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.opensearch;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.store.exception.StoreException;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import org.apache.hc.client5.http.auth.AuthScope;
|
||||
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
|
||||
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
|
||||
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
|
||||
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
|
||||
import org.apache.hc.core5.http.HttpHost;
|
||||
import org.apache.hc.core5.http.message.BasicHeader;
|
||||
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
|
||||
import org.apache.hc.core5.ssl.SSLContextBuilder;
|
||||
import org.opensearch.client.json.JsonData;
|
||||
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
|
||||
import org.opensearch.client.opensearch.OpenSearchClient;
|
||||
import org.opensearch.client.opensearch._types.ErrorCause;
|
||||
import org.opensearch.client.opensearch._types.InlineScript;
|
||||
import org.opensearch.client.opensearch._types.mapping.Property;
|
||||
import org.opensearch.client.opensearch._types.mapping.TextProperty;
|
||||
import org.opensearch.client.opensearch._types.mapping.TypeMapping;
|
||||
import org.opensearch.client.opensearch._types.query_dsl.Query;
|
||||
import org.opensearch.client.opensearch._types.query_dsl.ScriptScoreQuery;
|
||||
import org.opensearch.client.opensearch.core.BulkRequest;
|
||||
import org.opensearch.client.opensearch.core.BulkResponse;
|
||||
import org.opensearch.client.opensearch.core.SearchRequest;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
|
||||
import org.opensearch.client.transport.OpenSearchTransport;
|
||||
import org.opensearch.client.transport.endpoints.BooleanResponse;
|
||||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.security.KeyManagementException;
|
||||
import java.security.KeyStoreException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.stream.Collectors.toList;
|
||||
|
||||
/**
|
||||
* OpenSearch 向量存储
|
||||
*
|
||||
* @author songyinyin
|
||||
* @since 2024/8/10 下午8:31
|
||||
*/
|
||||
public class OpenSearchVectorStore extends DocumentStore {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(OpenSearchVectorStore.class);
|
||||
|
||||
private final OpenSearchClient client;
|
||||
|
||||
private final OpenSearchVectorStoreConfig config;
|
||||
|
||||
public OpenSearchVectorStore(OpenSearchVectorStoreConfig config) {
|
||||
this.config = config;
|
||||
HttpHost openSearchHost;
|
||||
try {
|
||||
openSearchHost = HttpHost.create(config.getServerUrl());
|
||||
} catch (URISyntaxException se) {
|
||||
log.error("[OpenSearch Exception]", se);
|
||||
throw new StoreException(se.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build();
|
||||
TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
|
||||
.setSslContext(sslContext)
|
||||
.setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
|
||||
.build();
|
||||
|
||||
OpenSearchTransport transport = ApacheHttpClient5TransportBuilder
|
||||
.builder(openSearchHost)
|
||||
.setMapper(new JacksonJsonpMapper())
|
||||
.setHttpClientConfigCallback(httpClientBuilder -> {
|
||||
|
||||
if (StringUtil.hasText(config.getApiKey())) {
|
||||
httpClientBuilder.setDefaultHeaders(singletonList(
|
||||
new BasicHeader("Authorization", "ApiKey " + config.getApiKey())
|
||||
));
|
||||
}
|
||||
|
||||
if (StringUtil.hasText(config.getUsername()) && StringUtil.hasText(config.getPassword())) {
|
||||
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(new AuthScope(openSearchHost),
|
||||
new UsernamePasswordCredentials(config.getUsername(), config.getPassword().toCharArray()));
|
||||
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
}
|
||||
|
||||
httpClientBuilder.setConnectionManager(PoolingAsyncClientConnectionManagerBuilder
|
||||
.create().setTlsStrategy(tlsStrategy).build());
|
||||
|
||||
return httpClientBuilder;
|
||||
})
|
||||
.build();
|
||||
|
||||
this.client = new OpenSearchClient(transport);
|
||||
try {
|
||||
client.ping();
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O OpenSearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
|
||||
throw new StoreException("OpenSearchClient init error", e);
|
||||
}
|
||||
}
|
||||
|
||||
public OpenSearchVectorStore(OpenSearchVectorStoreConfig config, OpenSearchClient client) {
|
||||
this.config = config;
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
private void createIndexIfNotExist(String indexName) {
|
||||
try {
|
||||
BooleanResponse response = client.indices().exists(c -> c.index(indexName));
|
||||
if (!response.value()) {
|
||||
log.info("[OpenSearch] Index {} not exists, creating...", indexName);
|
||||
client.indices().create(c -> c.index(indexName)
|
||||
.settings(s -> s.knn(true))
|
||||
.mappings(getDefaultMappings(this.getEmbeddingModel().dimensions())));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O OpenSearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private TypeMapping getDefaultMappings(int dimension) {
|
||||
Map<String, Property> properties = new HashMap<>(4);
|
||||
properties.put("content", Property.of(p -> p.text(TextProperty.of(t -> t))));
|
||||
properties.put("vector", Property.of(p -> p.knnVector(
|
||||
k -> k.dimension(dimension)
|
||||
)));
|
||||
return TypeMapping.of(c -> c.properties(properties));
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
createIndexIfNotExist(indexName);
|
||||
for (Document document : documents) {
|
||||
bulkBuilder.operations(op -> op.index(
|
||||
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
|
||||
);
|
||||
}
|
||||
bulk(bulkBuilder.build());
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
private void bulk(BulkRequest bulkRequest) {
|
||||
try {
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest);
|
||||
throwIfError(bulkResponse);
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O OpenSearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void throwIfError(BulkResponse bulkResponse) {
|
||||
if (bulkResponse.errors()) {
|
||||
for (BulkResponseItem item : bulkResponse.items()) {
|
||||
if (item.error() == null) {
|
||||
continue;
|
||||
}
|
||||
ErrorCause errorCause = item.error();
|
||||
throw new StoreException("type: " + errorCause.type() + "," + "reason: " + errorCause.reason());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
|
||||
for (Object id : ids) {
|
||||
bulkBuilder.operations(op -> op.delete(d -> d.index(indexName).id(id.toString())));
|
||||
}
|
||||
bulk(bulkBuilder.build());
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
BulkRequest.Builder bulkBuilder = new BulkRequest.Builder();
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
for (Document document : documents) {
|
||||
bulkBuilder.operations(op -> op.update(
|
||||
idx -> idx.index(indexName).id(document.getId().toString()).document(document))
|
||||
);
|
||||
}
|
||||
bulk(bulkBuilder.build());
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
Double minScore = wrapper.getMinScore();
|
||||
String indexName = options.getIndexNameOrDefault(config.getDefaultIndexName());
|
||||
|
||||
// https://aws.amazon.com/cn/blogs/china/use-aws-opensearch-knn-plug-in-to-implement-vector-retrieval/
|
||||
// boost 默认是 1,小于 1 会降低相关性: https://opensearch.org/docs/latest/query-dsl/specialized/script-score/#parameters
|
||||
ScriptScoreQuery scriptScoreQuery = ScriptScoreQuery.of(q -> q.minScore(minScore == null ? 0 : minScore.floatValue())
|
||||
.query(Query.of(qu -> qu.matchAll(m -> m)))
|
||||
.script(s -> s.inline(InlineScript.of(i -> i
|
||||
.source("knn_score")
|
||||
.lang("knn")
|
||||
.params("field", JsonData.of("vector"))
|
||||
.params("query_value", JsonData.of(wrapper.getVector()))
|
||||
.params("space_type", JsonData.of("cosinesimil"))
|
||||
))));
|
||||
|
||||
try {
|
||||
SearchResponse<Document> response = client.search(
|
||||
SearchRequest.of(s -> s.index(indexName)
|
||||
.query(n -> n.scriptScore(scriptScoreQuery))
|
||||
.size(wrapper.getMaxResults())),
|
||||
Document.class
|
||||
);
|
||||
return response.hits().hits().stream()
|
||||
.filter(s -> s.source() != null)
|
||||
.map(s -> {
|
||||
Document source = s.source();
|
||||
source.setScore(s.score());
|
||||
return source;
|
||||
})
|
||||
.collect(toList());
|
||||
} catch (IOException e) {
|
||||
log.error("[I/O OpenSearch Exception]", e);
|
||||
throw new StoreException(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.opensearch;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
/**
|
||||
* 连接 open search 配置:<a href="https://opensearch.org/docs/latest/clients/java/">opensearch-java</a>
|
||||
*
|
||||
* @author songyinyin
|
||||
* @since 2024/8/10 下午8:39
|
||||
*/
|
||||
public class OpenSearchVectorStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String serverUrl = "https://localhost:9200";
|
||||
|
||||
private String apiKey;
|
||||
|
||||
private String username;
|
||||
|
||||
private String password;
|
||||
|
||||
private String defaultIndexName = "easy-agents-default";
|
||||
|
||||
public String getServerUrl() {
|
||||
return serverUrl;
|
||||
}
|
||||
|
||||
public void setServerUrl(String serverUrl) {
|
||||
this.serverUrl = serverUrl;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getDefaultIndexName() {
|
||||
return defaultIndexName;
|
||||
}
|
||||
|
||||
public void setDefaultIndexName(String defaultIndexName) {
|
||||
this.defaultIndexName = defaultIndexName;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.serverUrl, this.apiKey)
|
||||
|| StringUtil.hasText(this.serverUrl, this.username, this.password);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.opensearch;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||
import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.VectorData;
|
||||
import com.easyagents.core.store.exception.StoreException;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author songyinyin
|
||||
* @since 2024/8/11 下午3:21
|
||||
*/
|
||||
public class OpenSearchVectorStoreTest {
|
||||
|
||||
@Test(expected = StoreException.class)
|
||||
public void test01() {
|
||||
OpenSearchVectorStore store = getOpenSearchVectorStore();
|
||||
|
||||
// https://opensearch.org/docs/latest/search-plugins/vector-search/#example
|
||||
List<Document> list = new ArrayList<>();
|
||||
Document doc1 = new Document();
|
||||
doc1.setId(1);
|
||||
doc1.setContent("test1");
|
||||
doc1.setVector(new float[]{5.2f, 4.4f});
|
||||
list.add(doc1);
|
||||
Document doc2 = new Document();
|
||||
doc2.setId(2);
|
||||
doc2.setContent("test2");
|
||||
doc2.setVector(new float[]{5.2f, 3.9f});
|
||||
list.add(doc2);
|
||||
Document doc3 = new Document();
|
||||
doc3.setId(3);
|
||||
doc3.setContent("test3");
|
||||
doc3.setVector(new float[]{4.9f, 3.4f});
|
||||
list.add(doc3);
|
||||
Document doc4 = new Document();
|
||||
doc4.setId(4);
|
||||
doc4.setContent("test4");
|
||||
doc4.setVector(new float[]{4.2f, 4.6f});
|
||||
list.add(doc4);
|
||||
Document doc5 = new Document();
|
||||
doc5.setId(5);
|
||||
doc5.setContent("test5");
|
||||
doc5.setVector(new float[]{3.3f, 4.5f});
|
||||
list.add(doc5);
|
||||
store.doStore(list, StoreOptions.DEFAULT);
|
||||
|
||||
// 可能要等一会 才能查出结果
|
||||
SearchWrapper searchWrapper = new SearchWrapper();
|
||||
searchWrapper.setVector(new float[]{5, 4});
|
||||
searchWrapper.setMaxResults(3);
|
||||
List<Document> documents = store.doSearch(searchWrapper, StoreOptions.DEFAULT);
|
||||
for (Document document : documents) {
|
||||
System.out.printf("id=%s, content=%s, vector=%s, metadata=%s\n",
|
||||
document.getId(), document.getContent(), Arrays.toString(document.getVector()), document.getMetadataMap());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static OpenSearchVectorStore getOpenSearchVectorStore() {
|
||||
OpenSearchVectorStoreConfig config = new OpenSearchVectorStoreConfig();
|
||||
config.setUsername("admin");
|
||||
config.setPassword("4_Pa46WQczS?");
|
||||
OpenSearchVectorStore store = new OpenSearchVectorStore(config);
|
||||
store.setEmbeddingModel(new EmbeddingModel() {
|
||||
@Override
|
||||
public VectorData embed(Document document, EmbeddingOptions options) {
|
||||
VectorData vectorData = new VectorData();
|
||||
vectorData.setVector(new float[]{0, 0});
|
||||
return vectorData;
|
||||
}
|
||||
});
|
||||
return store;
|
||||
}
|
||||
}
|
||||
41
easy-agents-store/easy-agents-store-pgvector/pom.xml
Normal file
41
easy-agents-store/easy-agents-store-pgvector/pom.xml
Normal file
@@ -0,0 +1,41 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-pgvector</name>
|
||||
<artifactId>easy-agents-store-pgvector</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.5</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.pgvector;
|
||||
|
||||
import org.postgresql.util.PGobject;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class PgvectorUtil {
|
||||
/**
|
||||
* 转化为vector.
|
||||
* 如果需要half vector或者sparse vector 对应实现即可
|
||||
* @param src 向量
|
||||
* @return
|
||||
* @throws SQLException
|
||||
*/
|
||||
public static PGobject toPgVector(double[] src) throws SQLException {
|
||||
PGobject vector = new PGobject();
|
||||
vector.setType("vector");
|
||||
if (src.length == 0) {
|
||||
vector.setValue("[]");
|
||||
return vector;
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder("[");
|
||||
for (double v : src) {
|
||||
sb.append(v);
|
||||
sb.append(",");
|
||||
}
|
||||
vector.setValue(sb.substring(0, sb.length() - 1) + "]");
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
public static double[] fromPgVector(String src) {
|
||||
if (src.equals("[]")) {
|
||||
return new double[0];
|
||||
}
|
||||
|
||||
String[] strs = src.substring(1, src.length() - 1).split(",");
|
||||
double[] output = new double[strs.length];
|
||||
for (int i = 0; i < strs.length; i++) {
|
||||
try {
|
||||
output[i] = Double.parseDouble(strs[i]);
|
||||
} catch (Exception ignore) {
|
||||
output[i] = 0;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.pgvector;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.postgresql.ds.PGSimpleDataSource;
|
||||
import org.postgresql.util.PGobject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
|
||||
public class PgvectorVectorStore extends DocumentStore {
|
||||
private static final Logger logger = LoggerFactory.getLogger(PgvectorVectorStore.class);
|
||||
public static final double DEFAULT_SIMILARITY_THRESHOLD = 0.3;
|
||||
private final PGSimpleDataSource dataSource;
|
||||
private final String defaultCollectionName;
|
||||
private final PgvectorVectorStoreConfig config;
|
||||
|
||||
|
||||
public PgvectorVectorStore(PgvectorVectorStoreConfig config) {
|
||||
dataSource = new PGSimpleDataSource();
|
||||
dataSource.setServerNames(new String[]{config.getHost() + ":" + config.getPort()});
|
||||
dataSource.setUser(config.getUsername());
|
||||
dataSource.setPassword(config.getPassword());
|
||||
dataSource.setDatabaseName(config.getDatabaseName());
|
||||
if (!config.getProperties().isEmpty()) {
|
||||
config.getProperties().forEach((k, v) -> {
|
||||
try {
|
||||
dataSource.setProperty(k, v);
|
||||
} catch (SQLException e) {
|
||||
logger.error("set pg property error", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
this.defaultCollectionName = config.getDefaultCollectionName();
|
||||
this.config = config;
|
||||
|
||||
// 异步初始化数据库
|
||||
new Thread(this::initDb).start();
|
||||
}
|
||||
|
||||
public void initDb() {
|
||||
// 启动的时候初始化向量表, 需要数据库支持pgvector插件
|
||||
// pg管理员需要在对应的库上执行 CREATE EXTENSION IF NOT EXISTS vector;
|
||||
if (config.isAutoCreateCollection()) {
|
||||
createCollection(defaultCollectionName);
|
||||
}
|
||||
}
|
||||
|
||||
private Connection getConnection() throws SQLException {
|
||||
Connection connection = dataSource.getConnection();
|
||||
connection.setAutoCommit(false);
|
||||
return connection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
|
||||
// 表名
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
|
||||
try (Connection connection = getConnection()) {
|
||||
PreparedStatement pstmt = connection.prepareStatement("insert into " + collectionName + " (id, content, vector, metadata) values (?, ?, ?, ?::jsonb)");
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> metadatas = doc.getMetadataMap();
|
||||
JSONObject jsonObject = JSON.parseObject(JSON.toJSONBytes(metadatas == null ? Collections.EMPTY_MAP : metadatas));
|
||||
pstmt.setString(1, String.valueOf(doc.getId()));
|
||||
pstmt.setString(2, doc.getContent());
|
||||
pstmt.setObject(3, PgvectorUtil.toPgVector(doc.getVectorAsDoubleArray()));
|
||||
pstmt.setString(4, jsonObject.toString());
|
||||
pstmt.addBatch();
|
||||
}
|
||||
|
||||
pstmt.executeBatch();
|
||||
connection.commit();
|
||||
} catch (SQLException e) {
|
||||
logger.error("store vector error", e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
private Boolean createCollection(String collectionName) {
|
||||
try (Connection connection = getConnection()) {
|
||||
try (CallableStatement statement = connection.prepareCall("CREATE TABLE IF NOT EXISTS " + collectionName +
|
||||
" (id varchar(100) PRIMARY KEY, content text, vector vector(" + config.getVectorDimension() + "), metadata jsonb)")) {
|
||||
statement.execute();
|
||||
}
|
||||
|
||||
// 默认情况下,pgvector 执行精确的最近邻搜索,从而提供完美的召回率. 可以通过索引来修改 pgvector 的搜索方式,以获得更好的性能。
|
||||
// By default, pgvector performs exact nearest neighbor search, which provides perfect recall.
|
||||
if (config.isUseHnswIndex()) {
|
||||
try (Statement stmt = connection.createStatement()) {
|
||||
stmt.execute("CREATE INDEX IF NOT EXISTS " + collectionName + "_vector_idx ON " + collectionName +
|
||||
" USING hnsw (vector vector_cosine_ops)");
|
||||
}
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.error("create collection error", e);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
StringBuilder sql = new StringBuilder("DELETE FROM " + options.getCollectionNameOrDefault(defaultCollectionName) + " WHERE id IN (");
|
||||
for (int i = 0; i < ids.size(); i++) {
|
||||
sql.append("?");
|
||||
if (i < ids.size() - 1) {
|
||||
sql.append(",");
|
||||
}
|
||||
}
|
||||
sql.append(")");
|
||||
|
||||
try (Connection connection = getConnection()) {
|
||||
PreparedStatement pstmt = connection.prepareStatement(sql.toString());
|
||||
ArrayList<?> list = new ArrayList<>(ids);
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
pstmt.setString(i + 1, (String) list.get(i));
|
||||
}
|
||||
|
||||
pstmt.executeUpdate();
|
||||
connection.commit();
|
||||
} catch (Exception e) {
|
||||
logger.error("delete document error: " + e, e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
|
||||
return StoreResult.success();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
|
||||
StringBuilder sql = new StringBuilder("select ");
|
||||
if (searchWrapper.isOutputVector()) {
|
||||
sql.append("id, vector, content, metadata");
|
||||
} else {
|
||||
sql.append("id, content, metadata");
|
||||
}
|
||||
|
||||
sql.append(" from ").append(options.getCollectionNameOrDefault(defaultCollectionName));
|
||||
sql.append(" where vector <=> ? < ? order by vector <=> ? LIMIT ?");
|
||||
|
||||
try (Connection connection = getConnection()){
|
||||
// 使用余弦距离计算最相似的文档
|
||||
PreparedStatement stmt = connection.prepareStatement(sql.toString());
|
||||
|
||||
PGobject vector = PgvectorUtil.toPgVector(searchWrapper.getVectorAsDoubleArray());
|
||||
stmt.setObject(1, vector);
|
||||
stmt.setObject(2, Optional.ofNullable(searchWrapper.getMinScore()).orElse(DEFAULT_SIMILARITY_THRESHOLD));
|
||||
stmt.setObject(3, vector);
|
||||
stmt.setObject(4, searchWrapper.getMaxResults());
|
||||
|
||||
ResultSet resultSet = stmt.executeQuery();
|
||||
List<Document> documents = new ArrayList<>();
|
||||
while (resultSet.next()) {
|
||||
Document doc = new Document();
|
||||
doc.setId(resultSet.getString("id"));
|
||||
doc.setContent(resultSet.getString("content"));
|
||||
doc.addMetadata(JSON.parseObject(resultSet.getString("metadata")));
|
||||
|
||||
if (searchWrapper.isOutputVector()) {
|
||||
String vectorStr = resultSet.getString("vector");
|
||||
doc.setVector(PgvectorUtil.fromPgVector(vectorStr));
|
||||
}
|
||||
|
||||
documents.add(doc);
|
||||
}
|
||||
|
||||
return documents;
|
||||
} catch (Exception e) {
|
||||
logger.error("Error searching in pgvector", e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
StringBuilder sql = new StringBuilder("UPDATE " + options.getCollectionNameOrDefault(defaultCollectionName) + " SET ");
|
||||
sql.append("content = ?, vector = ?, metadata = ?::jsonb WHERE id = ?");
|
||||
try (Connection connection = getConnection()) {
|
||||
PreparedStatement pstmt = connection.prepareStatement(sql.toString());
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> metadatas = doc.getMetadataMap();
|
||||
JSONObject metadataJson = JSON.parseObject(JSON.toJSONBytes(metadatas == null ? Collections.EMPTY_MAP : metadatas));
|
||||
pstmt.setString(1, doc.getContent());
|
||||
pstmt.setObject(2, PgvectorUtil.toPgVector(doc.getVectorAsDoubleArray()));
|
||||
pstmt.setString(3, metadataJson.toString());
|
||||
pstmt.setString(4, String.valueOf(doc.getId()));
|
||||
pstmt.addBatch();
|
||||
}
|
||||
|
||||
pstmt.executeUpdate();
|
||||
connection.commit();
|
||||
} catch (Exception e) {
|
||||
logger.error("Error update in pgvector", e);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.pgvector;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* postgreSQL访问配置
|
||||
* https://github.com/pgvector/pgvector
|
||||
*/
|
||||
public class PgvectorVectorStoreConfig implements DocumentStoreConfig {
|
||||
private String host;
|
||||
private int port = 5432;
|
||||
private String databaseName = "agent_vector";
|
||||
private String username;
|
||||
private String password;
|
||||
private Map<String, String> properties = new HashMap<>();
|
||||
private String defaultCollectionName;
|
||||
private boolean autoCreateCollection = true;
|
||||
private boolean useHnswIndex = false;
|
||||
private int vectorDimension = 1024;
|
||||
|
||||
public PgvectorVectorStoreConfig() {
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public String getDatabaseName() {
|
||||
return databaseName;
|
||||
}
|
||||
|
||||
public void setDatabaseName(String databaseName) {
|
||||
this.databaseName = databaseName;
|
||||
}
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
public boolean isAutoCreateCollection() {
|
||||
return autoCreateCollection;
|
||||
}
|
||||
|
||||
public void setAutoCreateCollection(boolean autoCreateCollection) {
|
||||
this.autoCreateCollection = autoCreateCollection;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public void setPort(int port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public Map<String, String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public void setProperties(Map<String, String> properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.host, this.username, this.password, this.databaseName);
|
||||
}
|
||||
|
||||
public int getVectorDimension() {
|
||||
return vectorDimension;
|
||||
}
|
||||
|
||||
public void setVectorDimension(int vectorDimension) {
|
||||
this.vectorDimension = vectorDimension;
|
||||
}
|
||||
|
||||
public boolean isUseHnswIndex() {
|
||||
return useHnswIndex;
|
||||
}
|
||||
|
||||
public void setUseHnswIndex(boolean useHnswIndex) {
|
||||
this.useHnswIndex = useHnswIndex;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.easyagents.store.pgvector;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.Maps;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class PgvectorDbTest {
|
||||
|
||||
@Test
|
||||
public void testInsert() {
|
||||
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
|
||||
config.setHost("127.0.0.1");
|
||||
config.setPort(5432);
|
||||
config.setDatabaseName("pgvector_test");
|
||||
config.setUsername("test");
|
||||
config.setPassword("123456");
|
||||
config.setVectorDimension(1024);
|
||||
config.setUseHnswIndex(true);
|
||||
config.setAutoCreateCollection(true);
|
||||
config.setDefaultCollectionName("test");
|
||||
|
||||
PgvectorVectorStore store = new PgvectorVectorStore(config);
|
||||
Document doc = new Document("测试数据");
|
||||
// 初始化 vector 为长度为 1024 的全是 1 的数组
|
||||
float[] vector = new float[1024];
|
||||
Arrays.fill(vector, 1.0f);
|
||||
|
||||
doc.setVector(vector);
|
||||
doc.setMetadataMap(Maps.of("test", "test"));
|
||||
store.store(doc);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsertMany() {
|
||||
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
|
||||
config.setHost("127.0.0.1");
|
||||
config.setPort(5432);
|
||||
config.setDatabaseName("pgvector_test");
|
||||
config.setUsername("test");
|
||||
config.setPassword("123456");
|
||||
config.setVectorDimension(1024);
|
||||
config.setUseHnswIndex(true);
|
||||
config.setAutoCreateCollection(true);
|
||||
config.setDefaultCollectionName("test");
|
||||
|
||||
PgvectorVectorStore store = new PgvectorVectorStore(config);
|
||||
List<Document> docs = new ArrayList<>(100);
|
||||
for (int i = 0; i < 100; i++) {
|
||||
Document doc = new Document("测试数据" + i);
|
||||
// 初始化 vector 为长度为 1024 的全是 1 的数组
|
||||
float[] vector = new float[1024];
|
||||
Arrays.fill(vector, (float) Math.random());
|
||||
|
||||
doc.setVector(vector);
|
||||
doc.setMetadataMap(Maps.of("test", "test" + i));
|
||||
docs.add(doc);
|
||||
}
|
||||
|
||||
store.store(docs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearch() {
|
||||
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
|
||||
config.setHost("127.0.0.1");
|
||||
config.setPort(5432);
|
||||
config.setDatabaseName("pgvector_test");
|
||||
config.setUsername("test");
|
||||
config.setPassword("123456");
|
||||
config.setVectorDimension(1024);
|
||||
config.setUseHnswIndex(true);
|
||||
config.setAutoCreateCollection(true);
|
||||
config.setDefaultCollectionName("test");
|
||||
PgvectorVectorStore store = new PgvectorVectorStore(config);
|
||||
|
||||
float[] vector = new float[1024];
|
||||
Arrays.fill(vector, 1.0f);
|
||||
|
||||
SearchWrapper searchWrapper = new SearchWrapper().text("测试数据");
|
||||
searchWrapper.setVector(vector);
|
||||
searchWrapper.setMinScore(0.0);
|
||||
searchWrapper.setOutputVector(true);
|
||||
List<Document> docs = store.search(searchWrapper);
|
||||
System.out.println(docs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdate() {
|
||||
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
|
||||
config.setHost("127.0.0.1");
|
||||
config.setPort(5432);
|
||||
config.setDatabaseName("pgvector_test");
|
||||
config.setUsername("test");
|
||||
config.setPassword("123456");
|
||||
config.setVectorDimension(1024);
|
||||
config.setUseHnswIndex(true);
|
||||
config.setAutoCreateCollection(true);
|
||||
config.setDefaultCollectionName("test");
|
||||
PgvectorVectorStore store = new PgvectorVectorStore(config);
|
||||
|
||||
Document document = new Document("测试数据");
|
||||
document.setId("145314895749100ae8306079519b3393");
|
||||
document.setMetadataMap(Maps.of("test", "test0"));
|
||||
float[] vector = new float[1024];
|
||||
Arrays.fill(vector, 1.1f);
|
||||
document.setVector(vector);
|
||||
StoreResult update = store.update(document);
|
||||
System.out.println(update);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDelete() {
|
||||
PgvectorVectorStoreConfig config = new PgvectorVectorStoreConfig();
|
||||
config.setHost("127.0.0.1");
|
||||
config.setPort(5432);
|
||||
config.setDatabaseName("pgvector_test");
|
||||
config.setUsername("test");
|
||||
config.setPassword("123456");
|
||||
config.setVectorDimension(1024);
|
||||
config.setUseHnswIndex(true);
|
||||
config.setAutoCreateCollection(true);
|
||||
config.setDefaultCollectionName("test");
|
||||
PgvectorVectorStore store = new PgvectorVectorStore(config);
|
||||
|
||||
StoreResult update = store.delete("145314895749100ae8306079519b3393","e83518d36b6d5de8199b40e3ef4e4ce1");
|
||||
System.out.println(update);
|
||||
}
|
||||
}
|
||||
28
easy-agents-store/easy-agents-store-qcloud/pom.xml
Normal file
28
easy-agents-store/easy-agents-store-qcloud/pom.xml
Normal file
@@ -0,0 +1,28 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-qcloud</name>
|
||||
<artifactId>easy-agents-store-qcloud</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.qcloud;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.model.client.HttpClient;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* doc https://cloud.tencent.com/document/product/1709/95121
|
||||
*/
|
||||
public class QCloudVectorStore extends DocumentStore {
|
||||
|
||||
private QCloudVectorStoreConfig config;
|
||||
|
||||
private final HttpClient httpUtil = new HttpClient();
|
||||
|
||||
|
||||
public QCloudVectorStore(QCloudVectorStoreConfig config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("database", config.getDatabase());
|
||||
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
|
||||
|
||||
|
||||
List<Map<String, Object>> payloadDocs = new ArrayList<>();
|
||||
for (Document vectorDocument : documents) {
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
if (vectorDocument.getMetadataMap() != null) {
|
||||
document.putAll(vectorDocument.getMetadataMap());
|
||||
}
|
||||
document.put("vector", vectorDocument.getVector());
|
||||
document.put("id", vectorDocument.getId());
|
||||
payloadDocs.add(document);
|
||||
}
|
||||
payloadMap.put("documents", payloadDocs);
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
httpUtil.post(config.getHost() + "/document/upsert", headers, payload);
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("database", config.getDatabase());
|
||||
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
|
||||
|
||||
Map<String, Object> documentIdsObj = new HashMap<>();
|
||||
documentIdsObj.put("documentIds", ids);
|
||||
payloadMap.put("query", documentIdsObj);
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
|
||||
httpUtil.post(config.getHost() + "/document/delete", headers, payload);
|
||||
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
|
||||
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("database", config.getDatabase());
|
||||
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
|
||||
|
||||
for (Document document : documents) {
|
||||
Map<String, Object> documentIdsObj = new HashMap<>();
|
||||
documentIdsObj.put("documentIds", Collections.singletonList(document.getId()));
|
||||
payloadMap.put("query", documentIdsObj);
|
||||
payloadMap.put("update", document.getMetadataMap());
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
httpUtil.post(config.getHost() + "/document/update", headers, payload);
|
||||
}
|
||||
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
|
||||
Map<String, String> headers = new HashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Bearer account=" + config.getAccount() + "&api_key=" + config.getApiKey());
|
||||
Map<String, Object> payloadMap = new HashMap<>();
|
||||
payloadMap.put("database", config.getDatabase());
|
||||
payloadMap.put("collection", options.getCollectionNameOrDefault(config.getDefaultCollectionName()));
|
||||
|
||||
Map<String, Object> searchMap = new HashMap<>();
|
||||
searchMap.put("vectors", Collections.singletonList(searchWrapper.getVector()));
|
||||
|
||||
if (searchWrapper.getMaxResults() != null) {
|
||||
searchMap.put("limit", searchWrapper.getMaxResults());
|
||||
}
|
||||
|
||||
payloadMap.put("search", searchMap);
|
||||
|
||||
|
||||
String payload = JSON.toJSONString(payloadMap);
|
||||
|
||||
// https://cloud.tencent.com/document/product/1709/95123
|
||||
String response = httpUtil.post(config.getHost() + "/document/search", headers, payload);
|
||||
if (StringUtil.noText(response)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<Document> result = new ArrayList<>();
|
||||
JSONObject rootObject = JSON.parseObject(response);
|
||||
int code = rootObject.getIntValue("code");
|
||||
if (code != 0) {
|
||||
LoggerFactory.getLogger(QCloudVectorStore.class).error("can not search in QCloudVectorStore, code:" + code + ", message: " + rootObject.getString("msg"));
|
||||
return null;
|
||||
}
|
||||
|
||||
JSONArray rootDocs = rootObject.getJSONArray("documents");
|
||||
for (int i = 0; i < rootDocs.size(); i++) {
|
||||
JSONArray docs = rootDocs.getJSONArray(i);
|
||||
for (int j = 0; j < docs.size(); j++) {
|
||||
JSONObject doc = docs.getJSONObject(j);
|
||||
Document vd = new Document();
|
||||
vd.setId(doc.getString("id"));
|
||||
doc.remove("id");
|
||||
vd.addMetadata(doc);
|
||||
result.add(vd);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.qcloud;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
public class QCloudVectorStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String host;
|
||||
private String apiKey;
|
||||
private String account;
|
||||
private String database;
|
||||
|
||||
private String defaultCollectionName;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public String getAccount() {
|
||||
return account;
|
||||
}
|
||||
|
||||
public void setAccount(String account) {
|
||||
this.account = account;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.host, this.apiKey, this.account, this.database, this.defaultCollectionName);
|
||||
}
|
||||
}
|
||||
|
||||
61
easy-agents-store/easy-agents-store-qdrant/pom.xml
Normal file
61
easy-agents-store/easy-agents-store-qdrant/pom.xml
Normal file
@@ -0,0 +1,61 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-qdrant</name>
|
||||
<artifactId>easy-agents-store-qdrant</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.grpc</groupId>
|
||||
<artifactId>grpc-stub</artifactId>
|
||||
<version>1.65.1</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.grpc</groupId>
|
||||
<artifactId>grpc-protobuf</artifactId>
|
||||
<version>1.65.1</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.grpc</groupId>
|
||||
<artifactId>grpc-netty-shaded</artifactId>
|
||||
<version>1.65.1</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.qdrant</groupId>
|
||||
<artifactId>client</artifactId>
|
||||
<version>1.14.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,186 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.qdrant;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.CollectionUtil;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import io.grpc.Grpc;
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.TlsChannelCredentials;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import io.qdrant.client.grpc.JsonWithInt;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import io.qdrant.client.grpc.Points.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.qdrant.client.ConditionFactory.matchKeyword;
|
||||
import static io.qdrant.client.PointIdFactory.id;
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorsFactory.vectors;
|
||||
import static io.qdrant.client.WithPayloadSelectorFactory.enable;
|
||||
|
||||
public class QdrantVectorStore extends DocumentStore {
|
||||
|
||||
private final QdrantVectorStoreConfig config;
|
||||
private final QdrantClient client;
|
||||
private final String defaultCollectionName;
|
||||
private boolean isCreateCollection = false;
|
||||
|
||||
public QdrantVectorStore(QdrantVectorStoreConfig config) throws IOException {
|
||||
this.config = config;
|
||||
this.defaultCollectionName = config.getDefaultCollectionName();
|
||||
String uri = config.getUri();
|
||||
int port = 6334;
|
||||
QdrantGrpcClient.Builder builder;
|
||||
if (StringUtil.hasText(config.getCaPath())) {
|
||||
ManagedChannel channel = Grpc.newChannelBuilder(
|
||||
uri,
|
||||
TlsChannelCredentials.newBuilder().trustManager(new File(config.getCaPath())).build()
|
||||
).build();
|
||||
builder = QdrantGrpcClient.newBuilder(channel, true);
|
||||
} else {
|
||||
if (uri.contains(":")) {
|
||||
uri = uri.split(":")[0];
|
||||
port = Integer.parseInt(uri.split(":")[1]);
|
||||
}
|
||||
builder = QdrantGrpcClient.newBuilder(uri, port, false);
|
||||
}
|
||||
if (StringUtil.hasText(config.getApiKey())) {
|
||||
builder.withApiKey(config.getApiKey());
|
||||
}
|
||||
this.client = new QdrantClient(builder.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
List<PointStruct> points = new ArrayList<>();
|
||||
int size = 1024;
|
||||
for (Document doc : documents) {
|
||||
size = doc.getVector().length;
|
||||
Map<String, JsonWithInt.Value> payload = new HashMap<>();
|
||||
payload.put("content", value(doc.getContent()));
|
||||
points.add(PointStruct.newBuilder()
|
||||
.setId(id(Long.parseLong(doc.getId().toString())))
|
||||
.setVectors(vectors(doc.getVector()))
|
||||
.putAllPayload(payload)
|
||||
.build());
|
||||
}
|
||||
try {
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
if (config.isAutoCreateCollection() && !isCreateCollection) {
|
||||
Boolean exists = client.collectionExistsAsync(collectionName).get();
|
||||
if (!exists) {
|
||||
client.createCollectionAsync(collectionName, Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.Cosine)
|
||||
.setSize(size)
|
||||
.build())
|
||||
.get();
|
||||
}
|
||||
} else {
|
||||
isCreateCollection = true;
|
||||
}
|
||||
if (CollectionUtil.hasItems(points)) {
|
||||
client.upsertAsync(collectionName, points).get();
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
} catch (Exception e) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
try {
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
List<PointId> pointIds = ids.stream()
|
||||
.map(id -> id((Long) id))
|
||||
.collect(Collectors.toList());
|
||||
client.deleteAsync(collectionName, pointIds).get();
|
||||
return StoreResult.success();
|
||||
} catch (Exception e) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
try {
|
||||
List<PointStruct> points = new ArrayList<>();
|
||||
for (Document doc : documents) {
|
||||
Map<String, JsonWithInt.Value> payload = new HashMap<>();
|
||||
payload.put("content", value(doc.getContent()));
|
||||
points.add(PointStruct.newBuilder()
|
||||
.setId(id(Long.parseLong(doc.getId().toString())))
|
||||
.setVectors(vectors(doc.getVector()))
|
||||
.putAllPayload(payload)
|
||||
.build());
|
||||
}
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
if (CollectionUtil.hasItems(points)) {
|
||||
client.upsertAsync(collectionName, points).get();
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
} catch (Exception e) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
List<Document> documents = new ArrayList<>();
|
||||
try {
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
QueryPoints.Builder query = QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setLimit(wrapper.getMaxResults())
|
||||
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(true).build())
|
||||
.setWithPayload(enable(true));
|
||||
if (wrapper.getVector() != null) {
|
||||
query.setQuery(nearest(wrapper.getVector()));
|
||||
}
|
||||
if (StringUtil.hasText(wrapper.getText())) {
|
||||
query.setFilter(Filter.newBuilder().addMust(matchKeyword("content", wrapper.getText())));
|
||||
}
|
||||
List<ScoredPoint> data = client.queryAsync(query.build()).get();
|
||||
for (ScoredPoint point : data) {
|
||||
Document doc = new Document();
|
||||
doc.setId(point.getId().getNum());
|
||||
doc.setVector(point.getVectors().getVector().getDataList());
|
||||
doc.setContent(point.getPayloadMap().get("content").getStringValue());
|
||||
documents.add(doc);
|
||||
}
|
||||
return documents;
|
||||
} catch (Exception e) {
|
||||
return documents;
|
||||
}
|
||||
}
|
||||
|
||||
public QdrantClient getClient() {
|
||||
return client;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.qdrant;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
public class QdrantVectorStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String uri;
|
||||
private String caPath;
|
||||
private String defaultCollectionName;
|
||||
private String apiKey;
|
||||
private boolean autoCreateCollection = true;
|
||||
|
||||
public String getUri() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
public void setUri(String uri) {
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
public String getCaPath() {
|
||||
return caPath;
|
||||
}
|
||||
|
||||
public void setCaPath(String caPath) {
|
||||
this.caPath = caPath;
|
||||
}
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public boolean isAutoCreateCollection() {
|
||||
return autoCreateCollection;
|
||||
}
|
||||
|
||||
public void setAutoCreateCollection(boolean autoCreateCollection) {
|
||||
this.autoCreateCollection = autoCreateCollection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.uri);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.easyagents.store.qdrant;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public class QdrantVectorStoreTest {
|
||||
|
||||
@Test
|
||||
public void testSaveVectors() throws Exception {
|
||||
QdrantVectorStore db = getDb();
|
||||
StoreOptions options = new StoreOptions();
|
||||
options.setCollectionName("test_collection1");
|
||||
List<Document> list = new ArrayList<>();
|
||||
Document doc1 = new Document();
|
||||
doc1.setId(1L);
|
||||
doc1.setContent("test1");
|
||||
doc1.setVector(new float[]{5.2f, 4.4f});
|
||||
list.add(doc1);
|
||||
Document doc2 = new Document();
|
||||
doc2.setId(2L);
|
||||
doc2.setContent("test2");
|
||||
doc2.setVector(new float[]{5.2f, 3.9f});
|
||||
list.add(doc2);
|
||||
Document doc3 = new Document();
|
||||
doc3.setId(3);
|
||||
doc3.setContent("test3");
|
||||
doc3.setVector(new float[]{4.9f, 3.4f});
|
||||
list.add(doc3);
|
||||
db.store(list, options);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQuery() throws Exception {
|
||||
QdrantVectorStore db = getDb();;
|
||||
StoreOptions options = new StoreOptions();
|
||||
options.setCollectionName("test_collection1");
|
||||
SearchWrapper search = new SearchWrapper();
|
||||
search.setVector(new float[]{5.2f, 3.9f});
|
||||
//search.setText("test1");
|
||||
search.setMaxResults(1);
|
||||
List<Document> record = db.search(search, options);
|
||||
System.out.println(record);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDelete() throws Exception {
|
||||
QdrantVectorStore db = getDb();
|
||||
StoreOptions options = new StoreOptions();
|
||||
options.setCollectionName("test_collection1");
|
||||
db.delete(Collections.singletonList(3L), options);
|
||||
}
|
||||
|
||||
private QdrantVectorStore getDb() throws Exception {
|
||||
QdrantVectorStoreConfig config = new QdrantVectorStoreConfig();
|
||||
config.setUri("localhost");
|
||||
config.setDefaultCollectionName("test_collection1");
|
||||
return new QdrantVectorStore(config);
|
||||
}
|
||||
}
|
||||
36
easy-agents-store/easy-agents-store-redis/pom.xml
Normal file
36
easy-agents-store/easy-agents-store-redis/pom.xml
Normal file
@@ -0,0 +1,36 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-redis</name>
|
||||
<artifactId>easy-agents-store-redis</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
<version>5.2.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.redis;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import kotlin.collections.ArrayDeque;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.JedisPooled;
|
||||
import redis.clients.jedis.Pipeline;
|
||||
import redis.clients.jedis.json.Path2;
|
||||
import redis.clients.jedis.search.FTCreateParams;
|
||||
import redis.clients.jedis.search.IndexDataType;
|
||||
import redis.clients.jedis.search.Query;
|
||||
import redis.clients.jedis.search.SearchResult;
|
||||
import redis.clients.jedis.search.schemafields.SchemaField;
|
||||
import redis.clients.jedis.search.schemafields.TextField;
|
||||
import redis.clients.jedis.search.schemafields.VectorField;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.FloatBuffer;
|
||||
import java.util.*;
|
||||
|
||||
public class RedisVectorStore extends DocumentStore {
|
||||
|
||||
protected final RedisVectorStoreConfig config;
|
||||
protected final JedisPooled jedis;
|
||||
protected final Set<String> redisIndexesCache = new HashSet<>();
|
||||
protected static final Logger logger = LoggerFactory.getLogger(RedisVectorStore.class);
|
||||
|
||||
|
||||
public RedisVectorStore(RedisVectorStoreConfig config) {
|
||||
this.config = config;
|
||||
this.jedis = new JedisPooled(
|
||||
URI.create(config.getUri())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
protected void createSchemaIfNecessary(String indexName) {
|
||||
if (redisIndexesCache.contains(indexName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 检查 indexName 是否存在
|
||||
Set<String> existIndexes = this.jedis.ftList();
|
||||
if (existIndexes != null && existIndexes.contains(indexName)) {
|
||||
redisIndexesCache.add(indexName);
|
||||
return;
|
||||
}
|
||||
|
||||
FTCreateParams ftCreateParams = FTCreateParams.createParams()
|
||||
.on(IndexDataType.JSON)
|
||||
.addPrefix(getPrefix(indexName));
|
||||
|
||||
jedis.ftCreate(indexName, ftCreateParams, schemaFields());
|
||||
redisIndexesCache.add(indexName);
|
||||
}
|
||||
|
||||
|
||||
protected Iterable<SchemaField> schemaFields() {
|
||||
Map<String, Object> vectorAttrs = new HashMap<>();
|
||||
//支持 COSINE: 余弦距离 , IP: 内积距离, L2: 欧几里得距离
|
||||
vectorAttrs.put("DISTANCE_METRIC", "COSINE");
|
||||
vectorAttrs.put("TYPE", "FLOAT32");
|
||||
vectorAttrs.put("DIM", this.getEmbeddingModel().dimensions());
|
||||
|
||||
List<SchemaField> fields = new ArrayList<>();
|
||||
fields.add(TextField.of(jsonPath("text")).as("text").weight(1.0));
|
||||
|
||||
fields.add(VectorField.builder()
|
||||
.fieldName(jsonPath("vector"))
|
||||
.algorithm(VectorField.VectorAlgorithm.HNSW)
|
||||
.attributes(vectorAttrs)
|
||||
.as("vector")
|
||||
.build());
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
protected String jsonPath(String field) {
|
||||
return "$." + field;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
String indexName = createIndexName(options);
|
||||
|
||||
if (StringUtil.noText(indexName)) {
|
||||
throw new IllegalStateException("IndexName is null or blank. please config the \"defaultCollectionName\" or store with designative collectionName.");
|
||||
}
|
||||
|
||||
createSchemaIfNecessary(indexName);
|
||||
|
||||
try (Pipeline pipeline = jedis.pipelined();) {
|
||||
for (Document document : documents) {
|
||||
java.util.Map<String, Object> fields = new HashMap<>();
|
||||
fields.put("text", document.getContent());
|
||||
fields.put("vector", document.getVector());
|
||||
|
||||
//put all metadata
|
||||
Map<String, Object> metadataMap = document.getMetadataMap();
|
||||
if (metadataMap != null) {
|
||||
fields.putAll(metadataMap);
|
||||
}
|
||||
|
||||
String key = getPrefix(indexName) + document.getId();
|
||||
pipeline.jsonSetWithEscape(key, Path2.of("$"), fields);
|
||||
}
|
||||
|
||||
List<Object> objects = pipeline.syncAndReturnAll();
|
||||
for (Object object : objects) {
|
||||
if (!object.equals("OK")) {
|
||||
logger.error("Could not store document: {}", object);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
String indexName = createIndexName(options);
|
||||
try (Pipeline pipeline = this.jedis.pipelined()) {
|
||||
for (Object id : ids) {
|
||||
String key = getPrefix(indexName) + id;
|
||||
pipeline.jsonDel(key);
|
||||
}
|
||||
|
||||
List<Object> objects = pipeline.syncAndReturnAll();
|
||||
for (Object object : objects) {
|
||||
if (!object.equals(1L)) {
|
||||
logger.error("Could not delete document: {}", object);
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return StoreResult.success();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
return doStore(documents, options);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper wrapper, StoreOptions options) {
|
||||
String indexName = createIndexName(options);
|
||||
|
||||
if (StringUtil.noText(indexName)) {
|
||||
throw new IllegalStateException("IndexName is null or blank. please config the \"defaultCollectionName\" or store with designative collectionName.");
|
||||
}
|
||||
|
||||
createSchemaIfNecessary(indexName);
|
||||
|
||||
// 创建查询向量
|
||||
byte[] vectorBytes = new byte[wrapper.getVector().length * 4];
|
||||
FloatBuffer floatBuffer = ByteBuffer.wrap(vectorBytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer();
|
||||
for (Float v : wrapper.getVector()) {
|
||||
floatBuffer.put(v);
|
||||
}
|
||||
|
||||
|
||||
List<String> returnFields = new ArrayList<>();
|
||||
returnFields.add("text");
|
||||
returnFields.add("vector");
|
||||
returnFields.add("score");
|
||||
|
||||
if (wrapper.getOutputFields() != null) {
|
||||
returnFields.addAll(wrapper.getOutputFields());
|
||||
}
|
||||
|
||||
// 使用 KNN 算法进行向量相似度搜索
|
||||
Query query = new Query("*=>[KNN " + wrapper.getMaxResults() + " @vector $BLOB AS score]")
|
||||
.addParam("BLOB", vectorBytes)
|
||||
.returnFields(returnFields.toArray(new String[0]))
|
||||
.setSortBy("score", true)
|
||||
.limit(0, wrapper.getMaxResults())
|
||||
.dialect(2);
|
||||
|
||||
int keyPrefixLen = this.getPrefix(indexName).length();
|
||||
|
||||
// 执行搜索
|
||||
SearchResult searchResult = jedis.ftSearch(indexName, query);
|
||||
List<redis.clients.jedis.search.Document> searchDocuments = searchResult.getDocuments();
|
||||
List<Document> documents = new ArrayDeque<>(searchDocuments.size());
|
||||
for (redis.clients.jedis.search.Document document : searchDocuments) {
|
||||
String id = document.getId().substring(keyPrefixLen);
|
||||
Document doc = new Document();
|
||||
doc.setId(id);
|
||||
doc.setContent(document.getString("text"));
|
||||
Object vector = document.get("vector");
|
||||
if (vector != null) {
|
||||
float[] doubles = JSON.parseObject(vector.toString(), float[].class);
|
||||
doc.setVector(doubles);
|
||||
}
|
||||
|
||||
if (wrapper.getOutputFields() != null) {
|
||||
for (String field : wrapper.getOutputFields()) {
|
||||
doc.addMetadata(field, document.getString(field));
|
||||
}
|
||||
}
|
||||
|
||||
double distance = 1.0d - similarityScore(document);
|
||||
// 相似度得分设置为0-1 , 0表示最不相似, 1表示最相似
|
||||
doc.setScore(1.0d - distance);
|
||||
documents.add(doc);
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
protected float similarityScore(redis.clients.jedis.search.Document doc) {
|
||||
return (2 - Float.parseFloat(doc.getString("score"))) / 2;
|
||||
}
|
||||
|
||||
|
||||
protected String createIndexName(StoreOptions options) {
|
||||
return options.getCollectionNameOrDefault(config.getDefaultCollectionName());
|
||||
}
|
||||
|
||||
@NotNull
|
||||
protected String getPrefix(String indexName) {
|
||||
return this.config.getStorePrefix() + indexName + ":";
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.redis;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
|
||||
public class RedisVectorStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String uri;
|
||||
|
||||
private String storePrefix = "docs:";
|
||||
private String defaultCollectionName;
|
||||
|
||||
|
||||
public String getUri() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
public void setUri(String uri) {
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
public String getStorePrefix() {
|
||||
return storePrefix;
|
||||
}
|
||||
|
||||
public void setStorePrefix(String storePrefix) {
|
||||
this.storePrefix = storePrefix;
|
||||
}
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.uri);
|
||||
}
|
||||
}
|
||||
36
easy-agents-store/easy-agents-store-vectorex/pom.xml
Normal file
36
easy-agents-store/easy-agents-store-vectorex/pom.xml
Normal file
@@ -0,0 +1,36 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-vectorex</name>
|
||||
<artifactId>easy-agents-store-vectorex</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.github.javpower</groupId>
|
||||
<artifactId>vectorex-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.vectorex;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import io.github.javpower.vectorexclient.VectorRexClient;
|
||||
import io.github.javpower.vectorexclient.builder.QueryBuilder;
|
||||
import io.github.javpower.vectorexclient.entity.MetricType;
|
||||
import io.github.javpower.vectorexclient.entity.ScalarField;
|
||||
import io.github.javpower.vectorexclient.entity.VectoRexEntity;
|
||||
import io.github.javpower.vectorexclient.entity.VectorFiled;
|
||||
import io.github.javpower.vectorexclient.req.CollectionDataAddReq;
|
||||
import io.github.javpower.vectorexclient.req.CollectionDataDelReq;
|
||||
import io.github.javpower.vectorexclient.req.VectoRexCollectionReq;
|
||||
import io.github.javpower.vectorexclient.res.DbData;
|
||||
import io.github.javpower.vectorexclient.res.ServerResponse;
|
||||
import io.github.javpower.vectorexclient.res.VectorSearchResult;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class VectoRexStore extends DocumentStore {
|
||||
|
||||
private final VectoRexStoreConfig config;
|
||||
private final VectorRexClient client;
|
||||
private final String defaultCollectionName;
|
||||
private boolean isCreateCollection = false;
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectoRexStore.class);
|
||||
|
||||
public VectoRexStore(VectoRexStoreConfig config) {
|
||||
this.config = config;
|
||||
this.defaultCollectionName = config.getDefaultCollectionName();
|
||||
this.client = new VectorRexClient(config.getUri(), config.getUsername(), config.getPassword());
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> dict = new HashMap<>();
|
||||
dict.put("id", String.valueOf(doc.getId()));
|
||||
dict.put("content", doc.getContent());
|
||||
dict.put("vector", doc.getVectorAsList());
|
||||
data.add(dict);
|
||||
}
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
if (config.isAutoCreateCollection() && !isCreateCollection) {
|
||||
ServerResponse<List<VectoRexEntity>> collections = client.getCollections();
|
||||
if (collections.getData() == null || collections.getData().stream().noneMatch(e -> e.getCollectionName().equals(collectionName))) {
|
||||
createCollection(collectionName);
|
||||
} else {
|
||||
isCreateCollection = true;
|
||||
}
|
||||
}
|
||||
for (Map<String, Object> map : data) {
|
||||
CollectionDataAddReq req = CollectionDataAddReq.builder().collectionName(collectionName).metadata(map).build();
|
||||
client.addCollectionData(req);
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
|
||||
private Boolean createCollection(String collectionName) {
|
||||
List<ScalarField> scalarFields = new ArrayList();
|
||||
ScalarField id = ScalarField.builder().name("id").isPrimaryKey(true).build();
|
||||
ScalarField content = ScalarField.builder().name("content").isPrimaryKey(false).build();
|
||||
scalarFields.add(id);
|
||||
scalarFields.add(content);
|
||||
List<VectorFiled> vectorFiles = new ArrayList();
|
||||
VectorFiled vector = VectorFiled.builder().name("vector").metricType(MetricType.FLOAT_COSINE_DISTANCE).dimensions(this.getEmbeddingModel().dimensions()).build();
|
||||
vectorFiles.add(vector);
|
||||
ServerResponse<Void> response = client.createCollection(VectoRexCollectionReq.builder().collectionName(collectionName).scalarFields(scalarFields).vectorFileds(vectorFiles).build());
|
||||
return response.isSuccess();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
for (Object id : ids) {
|
||||
CollectionDataDelReq req = CollectionDataDelReq.builder().collectionName(options.getCollectionNameOrDefault(defaultCollectionName)).id((String) id).build();
|
||||
ServerResponse<Void> response = client.deleteCollectionData(req);
|
||||
if (!response.isSuccess()) {
|
||||
return StoreResult.fail();
|
||||
}
|
||||
}
|
||||
return StoreResult.success();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
|
||||
ServerResponse<List<VectorSearchResult>> response = client.queryCollectionData(QueryBuilder.lambda(options.getCollectionNameOrDefault(defaultCollectionName))
|
||||
.vector("vector", Collections.singletonList(searchWrapper.getVectorAsList())).topK(searchWrapper.getMaxResults()));
|
||||
if (!response.isSuccess()) {
|
||||
logger.error("Error searching in VectoRex", response.getMsg());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<VectorSearchResult> data = response.getData();
|
||||
List<Document> documents = new ArrayList<>();
|
||||
for (VectorSearchResult result : data) {
|
||||
DbData dd = result.getData();
|
||||
Map<String, Object> metadata = dd.getMetadata();
|
||||
Document doc = new Document();
|
||||
doc.setId(result.getId());
|
||||
doc.setContent((String) metadata.get("content"));
|
||||
Object vectorObj = metadata.get("vector");
|
||||
if (vectorObj instanceof List) {
|
||||
//noinspection unchecked
|
||||
doc.setVector((List<Float>) vectorObj);
|
||||
}
|
||||
documents.add(doc);
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> dict = new HashMap<>();
|
||||
dict.put("id", String.valueOf(doc.getId()));
|
||||
dict.put("content", doc.getContent());
|
||||
dict.put("vector", doc.getVectorAsList());
|
||||
data.add(dict);
|
||||
}
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
for (Map<String, Object> map : data) {
|
||||
CollectionDataAddReq req = CollectionDataAddReq.builder().collectionName(collectionName).metadata(map).build();
|
||||
client.updateCollectionData(req);
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
public VectorRexClient getClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.vectorex;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
|
||||
public class VectoRexStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String uri;
|
||||
private String username;
|
||||
private String password;
|
||||
|
||||
private String defaultCollectionName;
|
||||
private boolean autoCreateCollection = true;
|
||||
|
||||
public String getUri() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
public void setUri(String uri) {
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
public boolean isAutoCreateCollection() {
|
||||
return autoCreateCollection;
|
||||
}
|
||||
|
||||
public void setAutoCreateCollection(boolean autoCreateCollection) {
|
||||
this.autoCreateCollection = autoCreateCollection;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.uri);
|
||||
}
|
||||
}
|
||||
41
easy-agents-store/easy-agents-store-vectorexdb/pom.xml
Normal file
41
easy-agents-store/easy-agents-store-vectorexdb/pom.xml
Normal file
@@ -0,0 +1,41 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store-vectorexdb</name>
|
||||
<artifactId>easy-agents-store-vectorexdb</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.github.javpower</groupId>
|
||||
<artifactId>vectorex-core</artifactId>
|
||||
<version>1.5.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-chat-openai</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.vectorex;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.core.util.CollectionUtil;
|
||||
import com.google.common.collect.Lists;
|
||||
import io.github.javpower.vectorex.keynote.core.DbData;
|
||||
import io.github.javpower.vectorex.keynote.core.VectorData;
|
||||
import io.github.javpower.vectorex.keynote.core.VectorSearchResult;
|
||||
import io.github.javpower.vectorex.keynote.model.MetricType;
|
||||
import io.github.javpower.vectorex.keynote.model.VectorFiled;
|
||||
import io.github.javpower.vectorexcore.VectoRexClient;
|
||||
import io.github.javpower.vectorexcore.entity.KeyValue;
|
||||
import io.github.javpower.vectorexcore.entity.ScalarField;
|
||||
import io.github.javpower.vectorexcore.entity.VectoRexEntity;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class VectoRexStore extends DocumentStore {
|
||||
|
||||
private final VectoRexStoreConfig config;
|
||||
private final VectoRexClient client;
|
||||
private final String defaultCollectionName;
|
||||
private boolean isCreateCollection=false;
|
||||
|
||||
public VectoRexStore(VectoRexStoreConfig config) {
|
||||
this.config = config;
|
||||
this.defaultCollectionName=config.getDefaultCollectionName();
|
||||
this.client = new VectoRexClient(config.getUri());
|
||||
}
|
||||
@Override
|
||||
public StoreResult doStore(List<Document> documents, StoreOptions options) {
|
||||
List<DbData> data=new ArrayList<>();
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> dict=new HashMap<>();
|
||||
dict.put("id",String.valueOf(doc.getId()));
|
||||
dict.put("content", doc.getContent());
|
||||
dict.put("vector", doc.getVector());
|
||||
DbData dbData=new DbData();
|
||||
dbData.setId(String.valueOf(doc.getId()));
|
||||
dbData.setMetadata(dict);
|
||||
VectorData vd=new VectorData(dbData.getId(),doc.getVector());
|
||||
vd.setName("vector");
|
||||
dbData.setVectorFiled(Lists.newArrayList(vd));
|
||||
data.add(dbData);
|
||||
}
|
||||
String collectionName = options.getCollectionNameOrDefault(defaultCollectionName);
|
||||
if(config.isAutoCreateCollection()&&!isCreateCollection){
|
||||
List<VectoRexEntity> collections = client.getCollections();
|
||||
if(CollectionUtil.noItems(collections)||collections.stream().noneMatch(e -> e.getCollectionName().equals(collectionName))){
|
||||
createCollection(collectionName);
|
||||
}else {
|
||||
isCreateCollection=true;
|
||||
}
|
||||
}
|
||||
if(CollectionUtil.hasItems(data)){
|
||||
client.getStore(collectionName).saveAll(data);
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
|
||||
private void createCollection(String collectionName) {
|
||||
VectorFiled vectorFiled = new VectorFiled();
|
||||
vectorFiled.setDimensions(this.getEmbeddingModel().dimensions());
|
||||
vectorFiled.setName("vector");
|
||||
vectorFiled.setMetricType(MetricType.FLOAT_COSINE_DISTANCE);
|
||||
VectoRexEntity entity=new VectoRexEntity();
|
||||
entity.setCollectionName(collectionName);
|
||||
List<KeyValue<String, VectorFiled>> vectorFiles=new ArrayList<>();
|
||||
vectorFiles.add(new KeyValue<>("vector",vectorFiled));
|
||||
List<KeyValue<String, ScalarField>> scalarFields=new ArrayList<>();
|
||||
ScalarField id = new ScalarField();
|
||||
id.setName("id");
|
||||
id.setIsPrimaryKey(true);
|
||||
scalarFields.add(new KeyValue<>("id",id));
|
||||
ScalarField content = new ScalarField();
|
||||
content.setName("content");
|
||||
content.setIsPrimaryKey(false);
|
||||
scalarFields.add(new KeyValue<>("content",content));
|
||||
entity.setVectorFileds(vectorFiles);
|
||||
entity.setScalarFields(scalarFields);
|
||||
client.createCollection(entity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doDelete(Collection<?> ids, StoreOptions options) {
|
||||
client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).deleteAll((List<String>) ids);
|
||||
return StoreResult.success();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> doSearch(SearchWrapper searchWrapper, StoreOptions options) {
|
||||
List<VectorSearchResult> data = client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).search("vector", searchWrapper.getVectorAsList(), searchWrapper.getMaxResults(), null);
|
||||
List<Document> documents = new ArrayList<>();
|
||||
for (VectorSearchResult result : data) {
|
||||
DbData dd = result.getData();
|
||||
Map<String, Object> metadata = dd.getMetadata();
|
||||
Document doc=new Document();
|
||||
doc.setId(result.getId());
|
||||
doc.setContent((String) metadata.get("content"));
|
||||
Object vectorObj = metadata.get("vector");
|
||||
if (vectorObj instanceof List) {
|
||||
//noinspection unchecked
|
||||
doc.setVector((List<Float>) vectorObj);
|
||||
}
|
||||
documents.add(doc);
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoreResult doUpdate(List<Document> documents, StoreOptions options) {
|
||||
if (documents == null || documents.isEmpty()) {
|
||||
return StoreResult.success();
|
||||
}
|
||||
for (Document doc : documents) {
|
||||
Map<String, Object> dict=new HashMap<>();
|
||||
dict.put("id",String.valueOf(doc.getId()));
|
||||
dict.put("content", doc.getContent());
|
||||
dict.put("vector", doc.getVector());
|
||||
DbData dbData=new DbData();
|
||||
dbData.setId(String.valueOf(doc.getId()));
|
||||
dbData.setMetadata(dict);
|
||||
VectorData vd=new VectorData(dbData.getId(),doc.getVector());
|
||||
vd.setName("vector");
|
||||
dbData.setVectorFiled(Lists.newArrayList(vd));
|
||||
client.getStore(options.getCollectionNameOrDefault(defaultCollectionName)).update(dbData);
|
||||
}
|
||||
return StoreResult.successWithIds(documents);
|
||||
}
|
||||
|
||||
public VectoRexClient getClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.vectorex;
|
||||
|
||||
import com.easyagents.core.store.DocumentStoreConfig;
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
|
||||
|
||||
public class VectoRexStoreConfig implements DocumentStoreConfig {
|
||||
|
||||
private String uri;
|
||||
private String defaultCollectionName;
|
||||
private boolean autoCreateCollection = true;
|
||||
|
||||
public String getUri() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
public void setUri(String uri) {
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
|
||||
public String getDefaultCollectionName() {
|
||||
return defaultCollectionName;
|
||||
}
|
||||
|
||||
public void setDefaultCollectionName(String defaultCollectionName) {
|
||||
this.defaultCollectionName = defaultCollectionName;
|
||||
}
|
||||
|
||||
public boolean isAutoCreateCollection() {
|
||||
return autoCreateCollection;
|
||||
}
|
||||
|
||||
public void setAutoCreateCollection(boolean autoCreateCollection) {
|
||||
this.autoCreateCollection = autoCreateCollection;
|
||||
}
|
||||
@Override
|
||||
public boolean checkAvailable() {
|
||||
return StringUtil.hasText(this.uri);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.easyagents.store.vectorex.test;
|
||||
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.easyagents.core.model.chat.ChatModel;
|
||||
import com.easyagents.core.store.SearchWrapper;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
|
||||
import com.easyagents.llm.openai.OpenAIChatModel;
|
||||
import com.easyagents.llm.openai.OpenAIChatConfig;
|
||||
import com.easyagents.store.vectorex.VectoRexStore;
|
||||
import com.easyagents.store.vectorex.VectoRexStoreConfig;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Test {
|
||||
public static void main(String[] args) {
|
||||
|
||||
OpenAIChatConfig openAILlmConfig= new OpenAIChatConfig();
|
||||
openAILlmConfig.setApiKey("");
|
||||
openAILlmConfig.setEndpoint("");
|
||||
openAILlmConfig.setModel("");
|
||||
|
||||
ChatModel chatModel = new OpenAIChatModel(openAILlmConfig);
|
||||
|
||||
|
||||
VectoRexStoreConfig config = new VectoRexStoreConfig();
|
||||
config.setDefaultCollectionName("test05");
|
||||
VectoRexStore store = new VectoRexStore(config);
|
||||
// store.setEmbeddingModel(chatModel);
|
||||
|
||||
Document document = new Document();
|
||||
document.setContent("你好");
|
||||
document.setId(1);
|
||||
store.store(document);
|
||||
|
||||
SearchWrapper sw = new SearchWrapper();
|
||||
sw.setText("你好");
|
||||
|
||||
List<Document> search = store.search(sw);
|
||||
System.out.println(search);
|
||||
|
||||
|
||||
StoreResult result = store.delete("1");
|
||||
System.out.println("-------delete-----" + result);
|
||||
search = store.search(sw);
|
||||
System.out.println(search);
|
||||
}
|
||||
}
|
||||
35
easy-agents-store/pom.xml
Normal file
35
easy-agents-store/pom.xml
Normal file
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-parent</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easy-agents-store</name>
|
||||
<artifactId>easy-agents-store</artifactId>
|
||||
|
||||
<packaging>pom</packaging>
|
||||
<modules>
|
||||
<module>easy-agents-store-qcloud</module>
|
||||
<module>easy-agents-store-aliyun</module>
|
||||
<module>easy-agents-store-chroma</module>
|
||||
<module>easy-agents-store-redis</module>
|
||||
<module>easy-agents-store-opensearch</module>
|
||||
<module>easy-agents-store-elasticsearch</module>
|
||||
<module>easy-agents-store-vectorex</module>
|
||||
<module>easy-agents-store-vectorexdb</module>
|
||||
<module>easy-agents-store-pgvector</module>
|
||||
<module>easy-agents-store-qdrant</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
</project>
|
||||
Reference in New Issue
Block a user