Merge branch 'dev' of https://gitea.wangzhiwen.top/0214/ALOps_sys1 into dev
commit
08c57cf67d
@ -0,0 +1,286 @@ |
||||
package com.alops.web.controller.LlmKnowledgeController; |
||||
|
||||
import com.alops.chat.entity.KnowledgeEntity; |
||||
import com.alops.chat.entity.UploadedFile; |
||||
import com.alops.chat.service.KnowledgeService; |
||||
import com.alops.chat.service.UploadedFileService; |
||||
import com.alops.common.constant.HttpStatus; |
||||
import com.alops.common.core.controller.BaseController; |
||||
import com.alops.common.core.domain.AjaxResult; |
||||
import com.alops.common.core.page.TableDataInfo; |
||||
import com.alops.common.config.RuoYiConfig; |
||||
import com.alops.common.utils.file.FileUploadUtils; |
||||
import com.alops.common.utils.file.MimeTypeUtils; |
||||
import io.swagger.annotations.ApiOperation; |
||||
import io.swagger.annotations.ApiParam; |
||||
import org.apache.poi.xwpf.usermodel.XWPFDocument; |
||||
import org.apache.poi.xwpf.usermodel.XWPFParagraph; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.security.access.prepost.PreAuthorize; |
||||
import org.springframework.web.bind.annotation.*; |
||||
import org.springframework.web.multipart.MultipartFile; |
||||
|
||||
import java.io.BufferedReader; |
||||
import java.io.IOException; |
||||
import java.io.InputStreamReader; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
import java.nio.file.StandardCopyOption; |
||||
import java.time.LocalDateTime; |
||||
import java.time.format.DateTimeFormatter; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* 文件上传控制器 |
||||
* 提供文件导入功能，将文本文档转化为知识条目 |
||||
*/ |
||||
@RestController |
||||
@RequestMapping("/QandA/knowledge/") |
||||
@CrossOrigin(origins = "*") |
||||
public class KnowLedgeController extends BaseController { |
||||
|
||||
@Autowired |
||||
private KnowledgeService knowledgeService; |
||||
|
||||
@Autowired |
||||
private UploadedFileService uploadedFileService; |
||||
|
||||
// 文件存储目录
|
||||
private static final String UPLOAD_DIR = "uploads/"; |
||||
|
||||
/** |
||||
* 上传并处理文本文档 |
||||
* |
||||
* @param file 上传的文件 |
||||
* @param source 文件来源描述 |
||||
* @return 处理结果 |
||||
*/ |
||||
|
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:uploadfile')") |
||||
@PostMapping("/upload") |
||||
@ApiOperation("上传并处理文本文档") |
||||
public AjaxResult uploadFile( |
||||
@RequestPart("file") |
||||
@ApiParam(value = "知识库文件", type = "file") MultipartFile file, |
||||
@ModelAttribute("source") String source) { |
||||
try { |
||||
String content; |
||||
|
||||
// 判断文件类型并读取内容
|
||||
if (file.getOriginalFilename() != null && file.getOriginalFilename().endsWith(".docx")) { |
||||
// 处理docx文件
|
||||
content = readDocxFile(file); |
||||
} else { |
||||
// 处理txt文件
|
||||
content = readTxtFile(file); |
||||
} |
||||
|
||||
// 使用优化后的文件上传工具保存文件到本地并记录信息到数据库
|
||||
// 上传文件并保持原始文件名，只允许 TEXT_EXTENSION 类型的文件
|
||||
String filePath = FileUploadUtils.uploadWithOriginalFilename( |
||||
RuoYiConfig.getProfile(), |
||||
file, |
||||
MimeTypeUtils.TEXT_EXTENSION); |
||||
|
||||
String fileExtension = getFileExtension(file.getOriginalFilename()); |
||||
UploadedFile uploadedFile = new UploadedFile(filePath, source, fileExtension, null); |
||||
uploadedFile = uploadedFileService.save(uploadedFile); |
||||
|
||||
// 获取保存后的文件ID
|
||||
Long fileId = uploadedFile.getId(); |
||||
|
||||
// 处理文件内容并添加到知识库，将文件ID作为keyfileid参数传递
|
||||
List<Long> knowledgeIds = knowledgeService.addKnowledgeWithFileId(content, source, fileId); |
||||
|
||||
// 更新上传文件信息（如果有知识条目被创建）
|
||||
if (!knowledgeIds.isEmpty()) { |
||||
// 可以在这里添加其他需要的更新逻辑
|
||||
} |
||||
|
||||
return AjaxResult.success("文件上传并处理成功"); |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
return AjaxResult.error("文件处理失败: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 读取docx文件的内容 |
||||
* |
||||
* @param file 上传的docx文件 |
||||
* @return 文件内容字符串 |
||||
* @throws IOException IO异常 |
||||
*/ |
||||
private String readDocxFile(MultipartFile file) throws IOException { |
||||
try (XWPFDocument document = new XWPFDocument(file.getInputStream())) { |
||||
StringBuilder content = new StringBuilder(); |
||||
// 遍历所有段落并提取文本
|
||||
for (XWPFParagraph paragraph : document.getParagraphs()) { |
||||
content.append(paragraph.getText()).append("\n"); |
||||
} |
||||
return content.toString(); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 读取txt文件的内容 |
||||
* |
||||
* @param file 上传的txt文件 |
||||
* @return 文件内容字符串 |
||||
* @throws IOException IO异常 |
||||
*/ |
||||
private String readTxtFile(MultipartFile file) throws IOException { |
||||
StringBuilder content = new StringBuilder(); |
||||
try (BufferedReader reader = new BufferedReader( |
||||
new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8))) { |
||||
String line; |
||||
while ((line = reader.readLine()) != null) { |
||||
content.append(line).append("\n"); |
||||
} |
||||
} |
||||
return content.toString(); |
||||
} |
||||
|
||||
|
||||
|
||||
/** |
||||
* 获取文件扩展名 |
||||
* |
||||
* @param filename 文件名 |
||||
* @return 文件扩展名 |
||||
*/ |
||||
private String getFileExtension(String filename) { |
||||
if (filename == null || filename.lastIndexOf(".") == -1) { |
||||
return ""; |
||||
} |
||||
return filename.substring(filename.lastIndexOf(".") + 1); |
||||
} |
||||
|
||||
/** |
||||
* 添加知识条目接口 |
||||
* |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @return 添加结果 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:upload')") |
||||
@PostMapping("/add-knowledge") |
||||
@ApiOperation("添加知识条目接口") |
||||
public AjaxResult addKnowledge( |
||||
@RequestParam String content, |
||||
@RequestParam String source) { |
||||
try { |
||||
knowledgeService.addKnowledge(content, source); |
||||
return AjaxResult.success("知识添加成功"); |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
return AjaxResult.error("知识添加失败: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
|
||||
/** |
||||
* 获取所有正式知识条目（分页） |
||||
* |
||||
* @return 正式知识条目列表 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:permquery')") |
||||
@GetMapping("/permanent-knowledge") |
||||
@ApiOperation("获取所有正式知识条目") |
||||
public TableDataInfo getPermanentKnowledge() { |
||||
startPage(); // 开启分页
|
||||
List<KnowledgeEntity> permanentEntries = knowledgeService.getPermanentEntries(); |
||||
return getDataTable(permanentEntries); |
||||
} |
||||
|
||||
/** |
||||
* 获取所有上传文件信息（分页） |
||||
* |
||||
* @return 上传文件信息列表 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:filelist')") |
||||
@GetMapping("/uploaded-files") |
||||
@ApiOperation("获取所有上传文档") |
||||
public TableDataInfo getUploadedFiles() { |
||||
startPage(); // 开启分页
|
||||
List<UploadedFile> uploadedFiles = uploadedFileService.findAll(); |
||||
return getDataTable(uploadedFiles); |
||||
} |
||||
|
||||
/** |
||||
* 更新知识条目接口 |
||||
* |
||||
* @param id 知识条目ID |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @return 更新结果 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:update')") |
||||
@PutMapping("/update-knowledge") |
||||
@ApiOperation("更新知识条目") |
||||
public AjaxResult updateKnowledge( |
||||
@RequestParam Long id, |
||||
@RequestParam String content, |
||||
@RequestParam String source) { |
||||
try { |
||||
boolean updated = knowledgeService.updateKnowledge(id, content, source); |
||||
if (updated) { |
||||
return AjaxResult.success("知识条目更新成功"); |
||||
} else { |
||||
return AjaxResult.error(HttpStatus.NOT_FOUND, "未找到指定的知识条目"); |
||||
} |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
return AjaxResult.error("知识条目更新失败: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 删除知识条目接口 |
||||
* |
||||
* @param id 知识条目ID |
||||
* @return 删除结果 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:del')") |
||||
@DeleteMapping("/knowledge/{id}") |
||||
@ApiOperation("删除知识条目接口") |
||||
public AjaxResult deleteKnowledge(@PathVariable Long id) { |
||||
try { |
||||
boolean deleted = knowledgeService.deleteKnowledge(id); |
||||
if (deleted) { |
||||
return AjaxResult.success("知识条目删除成功");} else { |
||||
return AjaxResult.error(HttpStatus.NOT_FOUND, "未找到指定的知识条目"); |
||||
} |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
return AjaxResult.error("知识条目删除失败: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 删除上传文件信息接口 |
||||
* |
||||
* @param id 上传文件ID |
||||
* @return 删除结果 |
||||
*/ |
||||
@ApiOperation("删除上传文件信息") |
||||
@PreAuthorize("@ss.hasPermi('QandA:knowledge:filedel')") |
||||
@DeleteMapping("/uploaded-files/{id}") |
||||
public AjaxResult deleteUploadedFile(@PathVariable Long id) { |
||||
try { |
||||
// 先删除关联的知识条目
|
||||
knowledgeService.deleteKnowledgeByFileId(id); |
||||
|
||||
// 再删除文件信息
|
||||
uploadedFileService.deleteById(id); |
||||
return AjaxResult.success("上传文件信息删除成功"); |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
return AjaxResult.error("上传文件信息删除失败: " + e.getMessage()); |
||||
} |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,139 @@ |
||||
package com.alops.web.controller.LlmKnowledgeController; |
||||
|
||||
import com.alops.chat.DTO.AnswerResponse; |
||||
import com.alops.chat.DTO.QuestionRequest; |
||||
import com.alops.chat.entity.KnowledgeEntity; |
||||
import com.alops.chat.service.AItokenService; |
||||
import com.alops.chat.service.KnowledgeService; |
||||
import com.alops.common.core.domain.AjaxResult; |
||||
import io.swagger.annotations.ApiOperation; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.beans.factory.annotation.Value; |
||||
import org.springframework.security.access.prepost.PreAuthorize; |
||||
import org.springframework.web.bind.annotation.*; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* 问答系统控制器 |
||||
* 提供基于知识库的问答接口 |
||||
*/ |
||||
|
||||
@RestController |
||||
@RequestMapping("/QandA/knowledge/") |
||||
@CrossOrigin(origins = "*") |
||||
public class QAController { |
||||
|
||||
@Autowired |
||||
private KnowledgeService knowledgeService; |
||||
|
||||
@Autowired |
||||
private AItokenService AItokenService; |
||||
|
||||
// 假数据答案
|
||||
@Value("${mock.qa.answer:这是默认的模拟答案}") |
||||
private String mockAnswer; |
||||
|
||||
// 假数据来源
|
||||
@Value("${mock.qa.sources:模拟来源1,模拟来源2}") |
||||
private String mockSources; |
||||
|
||||
/** |
||||
* POST方式问答接口 |
||||
* @param request 包含问题的请求对象 |
||||
* @return 答案响应 |
||||
*/ |
||||
@PreAuthorize("@ss.hasPermi('QandA:Aiassistant:ask')") |
||||
@PostMapping("/ask") |
||||
@ApiOperation("智能问答提问") |
||||
public AjaxResult askQuestion(@RequestBody QuestionRequest request) { |
||||
try { |
||||
// 检查是否使用假数据模式
|
||||
if (AItokenService.isMockEnabled()) { |
||||
System.out.println("使用假数据模式返回问答结果"); |
||||
String[] sources = mockSources.split(","); |
||||
AnswerResponse answerResponse = new AnswerResponse(mockAnswer, sources); |
||||
return AjaxResult.success("问答成功(假数据模式)", answerResponse); |
||||
} |
||||
|
||||
// 1. 查找相关知识（最相似的8条）
|
||||
List<KnowledgeEntity> similarKnowledge = knowledgeService.findSimilarKnowledge(request.getQuestion(), 8); |
||||
|
||||
// 如果没有找到相关知识，返回提示信息
|
||||
if (similarKnowledge.isEmpty()) { |
||||
// 创建一个专门的提示信息，说明知识库中未找到相关内容
|
||||
String noAnswerPrompt = "知识库中没有找到与您的问题直接相关的内容。您的问题可能是关于 \"" + |
||||
request.getQuestion() + "\"。请确认问题表述是否准确，或者尝试重新表述问题。" + |
||||
"如果这是一个重要的问题，请联系管理员添加相关知识。"; |
||||
|
||||
AnswerResponse answerResponse = new AnswerResponse(noAnswerPrompt, new String[0]); |
||||
return AjaxResult.success(noAnswerPrompt, answerResponse); |
||||
} |
||||
|
||||
// 2. 生成上下文
|
||||
String context = knowledgeService.generateContext(similarKnowledge); |
||||
|
||||
// 检查上下文是否为空
|
||||
if (context == null || context.trim().isEmpty()) { |
||||
String errorMessage = "系统未能从知识库中提取有效上下文信息。"; |
||||
AnswerResponse errorResponse = new AnswerResponse(errorMessage, new String[0]); |
||||
return AjaxResult.error(errorMessage, errorResponse); |
||||
} |
||||
|
||||
// 3. 构造消息列表并调用Qwen3生成答案
|
||||
List<Map<String, String>> messages = new ArrayList<>(); |
||||
Map<String, String> systemMessage = new HashMap<>(); |
||||
systemMessage.put("role", "system"); |
||||
systemMessage.put("content", "你是一个智能运维助手，根据提供的上下文回答问题。如果上下文中没有相关信息，请说明无法基于提供的上下文回答问题。"); |
||||
messages.add(systemMessage); |
||||
|
||||
Map<String, String> userMessage = new HashMap<>(); |
||||
userMessage.put("role", "user"); |
||||
userMessage.put("content", "上下文信息：\n" + context + "\n\n请根据以上上下文信息回答问题：" + request.getQuestion()); |
||||
messages.add(userMessage); |
||||
|
||||
String answer = AItokenService.generateAnswer(messages); |
||||
|
||||
// 检查生成的答案是否为空或默认加载信息
|
||||
if (answer == null || answer.trim().isEmpty()) { |
||||
String errorMessage = "系统未能生成有效答案，请稍后再试。"; |
||||
AnswerResponse errorResponse = new AnswerResponse(errorMessage, new String[0]); |
||||
return AjaxResult.error(errorMessage, errorResponse); |
||||
} |
||||
|
||||
// 检查是否是模型加载信息
|
||||
if (answer.contains("模型正在加载中")) { |
||||
AnswerResponse answerResponse = new AnswerResponse(answer, new String[0]); |
||||
return AjaxResult.error("模型正在加载中", answerResponse); |
||||
} |
||||
|
||||
// 检查是否是连接错误信息
|
||||
if (answer.contains("无法连接到Ollama服务")) { |
||||
AnswerResponse answerResponse = new AnswerResponse(answer, new String[0]); |
||||
return AjaxResult.error("服务连接失败", answerResponse); |
||||
} |
||||
|
||||
// 4. 提取来源信息
|
||||
String[] sources = similarKnowledge.stream() |
||||
.map(KnowledgeEntity::getSource) |
||||
.distinct() |
||||
.toArray(String[]::new); |
||||
|
||||
AnswerResponse answerResponse = new AnswerResponse(answer, sources); |
||||
return AjaxResult.success("问答成功", answerResponse); |
||||
} catch (Exception e) { |
||||
e.printStackTrace(); |
||||
AnswerResponse errorResponse = new AnswerResponse( |
||||
"系统处理问题时发生错误，请稍后再试。", new String[0]); |
||||
return AjaxResult.error("系统处理问题时发生错误，请稍后再试。", errorResponse); |
||||
} |
||||
} |
||||
|
||||
|
||||
|
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,152 @@ |
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" |
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
<parent> |
||||
<groupId>com.alops</groupId> |
||||
<artifactId>alops</artifactId> |
||||
<version>1.0.0</version> |
||||
<relativePath>../pom.xml</relativePath> |
||||
</parent> |
||||
<build> |
||||
<plugins> |
||||
<plugin> |
||||
<groupId>org.apache.maven.plugins</groupId> |
||||
<artifactId>maven-compiler-plugin</artifactId> |
||||
<configuration> |
||||
<source>9</source> |
||||
<target>9</target> |
||||
</configuration> |
||||
</plugin> |
||||
</plugins> |
||||
</build> |
||||
<modelVersion>4.0.0</modelVersion> |
||||
|
||||
<artifactId>alops-chat</artifactId> |
||||
|
||||
<description> |
||||
common通用工具 |
||||
</description> |
||||
|
||||
<dependencies> |
||||
<!-- org.snmp4j --> |
||||
<dependency> |
||||
<groupId>org.snmp4j</groupId> |
||||
<artifactId>snmp4j</artifactId> |
||||
<version>3.9.6</version> |
||||
</dependency> |
||||
|
||||
<!-- Spring框架基本的核心工具 --> |
||||
<dependency> |
||||
<groupId>org.springframework</groupId> |
||||
<artifactId>spring-context-support</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- SpringWeb模块 --> |
||||
<dependency> |
||||
<groupId>org.springframework</groupId> |
||||
<artifactId>spring-web</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- spring security 安全认证 --> |
||||
<dependency> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-security</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- pagehelper 分页插件 --> |
||||
<dependency> |
||||
<groupId>com.github.pagehelper</groupId> |
||||
<artifactId>pagehelper-spring-boot-starter</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- 自定义验证注解 --> |
||||
<dependency> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-validation</artifactId> |
||||
</dependency> |
||||
|
||||
<!--常用工具类 --> |
||||
<dependency> |
||||
<groupId>org.apache.commons</groupId> |
||||
<artifactId>commons-lang3</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- JSON工具类 --> |
||||
<dependency> |
||||
<groupId>com.fasterxml.jackson.core</groupId> |
||||
<artifactId>jackson-databind</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- 阿里JSON解析器 --> |
||||
<dependency> |
||||
<groupId>com.alibaba.fastjson2</groupId> |
||||
<artifactId>fastjson2</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- io常用工具类 --> |
||||
<dependency> |
||||
<groupId>commons-io</groupId> |
||||
<artifactId>commons-io</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- excel工具 --> |
||||
<dependency> |
||||
<groupId>org.apache.poi</groupId> |
||||
<artifactId>poi-ooxml</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- yml解析器 --> |
||||
<dependency> |
||||
<groupId>org.yaml</groupId> |
||||
<artifactId>snakeyaml</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- Token生成与解析--> |
||||
<dependency> |
||||
<groupId>io.jsonwebtoken</groupId> |
||||
<artifactId>jjwt</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- Jaxb --> |
||||
<dependency> |
||||
<groupId>javax.xml.bind</groupId> |
||||
<artifactId>jaxb-api</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- redis 缓存操作 --> |
||||
<dependency> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-data-redis</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- pool 对象池 --> |
||||
<dependency> |
||||
<groupId>org.apache.commons</groupId> |
||||
<artifactId>commons-pool2</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- 解析客户端操作系统、浏览器等 --> |
||||
<dependency> |
||||
<groupId>eu.bitwalker</groupId> |
||||
<artifactId>UserAgentUtils</artifactId> |
||||
</dependency> |
||||
|
||||
<!-- servlet包 --> |
||||
<dependency> |
||||
<groupId>javax.servlet</groupId> |
||||
<artifactId>javax.servlet-api</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.projectlombok</groupId> |
||||
<artifactId>lombok</artifactId> |
||||
<version>1.18.36</version> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-data-jpa</artifactId> |
||||
</dependency> |
||||
|
||||
</dependencies> |
||||
|
||||
</project> |
||||
@ -0,0 +1,34 @@ |
||||
// dto/AnswerResponse.java
|
||||
package com.alops.chat.DTO; |
||||
|
||||
public class AnswerResponse { |
||||
|
||||
private String answer; |
||||
private String[] sources; |
||||
|
||||
public AnswerResponse() {} |
||||
|
||||
public AnswerResponse(String answer, String[] sources) { |
||||
|
||||
this.answer = answer; |
||||
this.sources = sources; |
||||
} |
||||
|
||||
|
||||
|
||||
public String getAnswer() { |
||||
return answer; |
||||
} |
||||
|
||||
public void setAnswer(String answer) { |
||||
this.answer = answer; |
||||
} |
||||
|
||||
public String[] getSources() { |
||||
return sources; |
||||
} |
||||
|
||||
public void setSources(String[] sources) { |
||||
this.sources = sources; |
||||
} |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
// dto/EmbeddingResponse.java
|
||||
package com.alops.chat.DTO; |
||||
|
||||
import java.util.List; |
||||
|
||||
public class EmbeddingResponse { |
||||
private boolean success; |
||||
private List<Float> vector; |
||||
private int dimension; |
||||
private String errorMessage; // 添加错误信息字段
|
||||
|
||||
public EmbeddingResponse() {} |
||||
|
||||
public EmbeddingResponse(boolean success, List<Float> vector, int dimension) { |
||||
this.success = success; |
||||
this.vector = vector; |
||||
this.dimension = dimension; |
||||
} |
||||
|
||||
// 新增构造函数，包含错误信息
|
||||
public EmbeddingResponse(boolean success, List<Float> vector, int dimension, String errorMessage) { |
||||
this.success = success; |
||||
this.vector = vector; |
||||
this.dimension = dimension; |
||||
this.errorMessage = errorMessage; |
||||
} |
||||
|
||||
// Getter和Setter方法
|
||||
public boolean isSuccess() { |
||||
return success; |
||||
} |
||||
|
||||
public void setSuccess(boolean success) { |
||||
this.success = success; |
||||
} |
||||
|
||||
public List<Float> getVector() { |
||||
return vector; |
||||
} |
||||
|
||||
public void setVector(List<Float> vector) { |
||||
this.vector = vector; |
||||
} |
||||
|
||||
public int getDimension() { |
||||
return dimension; |
||||
} |
||||
|
||||
public void setDimension(int dimension) { |
||||
this.dimension = dimension; |
||||
} |
||||
|
||||
// 错误信息的Getter和Setter方法
|
||||
public String getErrorMessage() { |
||||
return errorMessage; |
||||
} |
||||
|
||||
public void setErrorMessage(String errorMessage) { |
||||
this.errorMessage = errorMessage; |
||||
} |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
// dto/QuestionRequest.java
|
||||
package com.alops.chat.DTO; |
||||
|
||||
public class QuestionRequest { |
||||
private String question; |
||||
|
||||
public QuestionRequest() {} |
||||
|
||||
public QuestionRequest(String question) { |
||||
this.question = question; |
||||
} |
||||
|
||||
public String getQuestion() { |
||||
return question; |
||||
} |
||||
|
||||
public void setQuestion(String question) { |
||||
this.question = question; |
||||
} |
||||
} |
||||
@ -0,0 +1,94 @@ |
||||
// entity/KnowledgeEntry.java
|
||||
package com.alops.chat.entity; |
||||
|
||||
|
||||
import lombok.Data; |
||||
|
||||
import javax.persistence.*; |
||||
import java.time.LocalDateTime; |
||||
|
||||
/** |
||||
* 知识条目实体类 |
||||
* 用于存储知识库中的条目信息 |
||||
*/ |
||||
@Entity |
||||
@Table(name = "knowledge_entries") |
||||
@Data |
||||
public class KnowledgeEntity { |
||||
|
||||
// 主键，自动生成
|
||||
@Id |
||||
@GeneratedValue(strategy = GenerationType.IDENTITY) |
||||
private Long id; |
||||
|
||||
// 知识内容，使用TEXT类型存储
|
||||
@Column(columnDefinition = "TEXT") |
||||
private String content; |
||||
|
||||
// 向量嵌入，使用Lob注解存储大型字节数组
|
||||
@Lob |
||||
private byte[] embedding; |
||||
|
||||
// 知识来源
|
||||
private String source; |
||||
|
||||
// 创建时间，实体被创建时自动设置
|
||||
@Column(name = "createtime") |
||||
private LocalDateTime createtime; |
||||
|
||||
// 更新时间，实体每次更新时自动设置
|
||||
@Column(name = "updatetime") |
||||
private LocalDateTime updatetime; |
||||
|
||||
// 对应的files的id
|
||||
@Column(name = "keyfileid") |
||||
private Integer keyfileid;; |
||||
|
||||
// 默认构造方法
|
||||
public KnowledgeEntity() { |
||||
} |
||||
|
||||
/** |
||||
* 构造方法 |
||||
* @param content 知识内容 |
||||
* @param embedding 向量嵌入 |
||||
* @param source 知识来源 |
||||
*/ |
||||
public KnowledgeEntity(String content, byte[] embedding, String source,int keyfileid) { |
||||
this.content = content; |
||||
this.embedding = embedding; |
||||
this.source = source; |
||||
this.createtime = LocalDateTime.now(); |
||||
this.updatetime = LocalDateTime.now(); |
||||
this.keyfileid =keyfileid; // 默认为正式条目
|
||||
} |
||||
|
||||
/** |
||||
* 构造方法（用于临时条目） |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @param keyfileid 对应的id |
||||
*/ |
||||
public KnowledgeEntity(String content, String source, int keyfileid) { |
||||
this.content = content; |
||||
this.source = source; |
||||
this.createtime = LocalDateTime.now(); |
||||
this.updatetime = LocalDateTime.now(); |
||||
this.keyfileid = keyfileid; |
||||
} |
||||
|
||||
|
||||
|
||||
// JPA生命周期回调方法，在实体被更新时自动设置更新时间
|
||||
@PreUpdate |
||||
public void onPreUpdate() { |
||||
this.updatetime = LocalDateTime.now(); |
||||
} |
||||
|
||||
// JPA生命周期回调方法，在实体被持久化之前自动设置创建时间
|
||||
@PrePersist |
||||
public void onPrePersist() { |
||||
this.createtime = LocalDateTime.now(); |
||||
this.updatetime = LocalDateTime.now(); |
||||
} |
||||
} |
||||
@ -0,0 +1,60 @@ |
||||
package com.alops.chat.entity; |
||||
|
||||
|
||||
import lombok.Data; |
||||
|
||||
import javax.persistence.*; |
||||
import java.time.LocalDateTime; |
||||
|
||||
/** |
||||
* 上传文件信息实体类 |
||||
* 用于存储上传文件的信息，包括路径、时间、格式等 |
||||
*/ |
||||
@Entity |
||||
@Table(name = "uploaded_files") |
||||
@Data |
||||
public class UploadedFile { |
||||
// 主键，自动生成
|
||||
@Id |
||||
@GeneratedValue(strategy = GenerationType.IDENTITY) |
||||
private Long id; |
||||
|
||||
// 文件保存路径
|
||||
@Column(name = "file_path") |
||||
private String filePath; |
||||
|
||||
// 上传时间
|
||||
@Column(name = "upload_time") |
||||
private LocalDateTime uploadTime; |
||||
|
||||
// 文件来源名称
|
||||
private String source; |
||||
|
||||
// 文件格式（扩展名）
|
||||
private String format; |
||||
|
||||
|
||||
|
||||
// 默认构造方法
|
||||
public UploadedFile() {} |
||||
|
||||
/** |
||||
* 构造方法 |
||||
* @param filePath 文件保存路径 |
||||
* @param source 文件来源名称 |
||||
* @param format 文件格式 |
||||
* @param knowledgeId 对应的知识条目ID |
||||
*/ |
||||
public UploadedFile(String filePath, String source, String format, Long knowledgeId) { |
||||
this.filePath = filePath; |
||||
this.source = source; |
||||
this.format = format; |
||||
this.uploadTime = LocalDateTime.now(); |
||||
} |
||||
|
||||
// JPA生命周期回调方法，在实体被持久化之前自动设置上传时间
|
||||
@PrePersist |
||||
public void onPrePersist() { |
||||
this.uploadTime = LocalDateTime.now(); |
||||
} |
||||
} |
||||
@ -0,0 +1,55 @@ |
||||
// repository/KnowledgeRepository.java
|
||||
package com.alops.chat.repository; |
||||
|
||||
import com.alops.chat.entity.KnowledgeEntity; |
||||
import org.springframework.data.jpa.repository.JpaRepository; |
||||
import org.springframework.data.jpa.repository.Modifying; |
||||
import org.springframework.data.jpa.repository.Query; |
||||
import org.springframework.data.repository.query.Param; |
||||
import org.springframework.stereotype.Repository; |
||||
import org.springframework.transaction.annotation.Transactional; |
||||
|
||||
import java.util.List; |
||||
|
||||
/** |
||||
* 知识条目数据访问接口 |
||||
* 提供对知识条目的增删改查操作 |
||||
*/ |
||||
@Repository |
||||
public interface KnowledgeRepository extends JpaRepository<KnowledgeEntity, Long> { |
||||
|
||||
/** |
||||
* 根据关键词模糊搜索知识条目 |
||||
* @param keyword 搜索关键词 |
||||
* @return 匹配的知识条目列表 |
||||
*/ |
||||
@Query("SELECT k FROM KnowledgeEntity k WHERE k.content LIKE %:keyword%") |
||||
List<KnowledgeEntity> findByContentContaining(@Param("keyword") String keyword); |
||||
|
||||
/** |
||||
* 根据多个关键词模糊搜索知识条目 |
||||
* @param keyword1 第一个关键词 |
||||
* @param keyword2 第二个关键词 |
||||
* @param keyword3 第三个关键词 |
||||
* @return 匹配的知识条目列表 |
||||
*/ |
||||
@Query("SELECT k FROM KnowledgeEntity k WHERE " + |
||||
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword1, '%')) OR " + |
||||
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword2, '%')) OR " + |
||||
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword3, '%'))") |
||||
List<KnowledgeEntity> findByMultipleKeywords( |
||||
@Param("keyword1") String keyword1, |
||||
@Param("keyword2") String keyword2, |
||||
@Param("keyword3") String keyword3); |
||||
|
||||
/** |
||||
* 根据文件ID删除知识条目 |
||||
* @param keyfileid 文件ID |
||||
* @return 删除的条目数量 |
||||
*/ |
||||
@Modifying |
||||
@Transactional |
||||
@Query("DELETE FROM KnowledgeEntity k WHERE k.keyfileid = :keyfileid") |
||||
int deleteByKeyfileid(@Param("keyfileid") Integer keyfileid); |
||||
|
||||
} |
||||
@ -0,0 +1,14 @@ |
||||
package com.alops.chat.repository; |
||||
|
||||
import com.alops.chat.entity.UploadedFile; |
||||
import org.springframework.data.jpa.repository.JpaRepository; |
||||
import org.springframework.stereotype.Repository; |
||||
|
||||
/** |
||||
* 上传文件信息Repository接口 |
||||
* 提供对UploadedFile实体的基本CRUD操作 |
||||
*/ |
||||
@Repository |
||||
public interface UploadedFileRepository extends JpaRepository<UploadedFile, Long> { |
||||
// JpaRepository已经提供了基本的CRUD操作，可以根据需要添加自定义查询方法
|
||||
} |
||||
@ -0,0 +1,750 @@ |
||||
// service/Qwen3Service.java
|
||||
package com.alops.chat.service; |
||||
import com.fasterxml.jackson.core.type.TypeReference; |
||||
import com.fasterxml.jackson.databind.JsonNode; |
||||
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
import com.alops.chat.DTO.EmbeddingResponse; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.beans.factory.annotation.Value; |
||||
import org.springframework.http.HttpEntity; |
||||
import org.springframework.http.HttpHeaders; |
||||
import org.springframework.http.MediaType; |
||||
import org.springframework.http.ResponseEntity; |
||||
import org.springframework.stereotype.Service; |
||||
import org.springframework.web.client.ResourceAccessException; |
||||
import org.springframework.web.client.RestTemplate; |
||||
|
||||
import java.net.InetSocketAddress; |
||||
import java.net.Socket; |
||||
import java.util.*; |
||||
|
||||
@Service |
||||
public class AItokenService { |
||||
|
||||
@Autowired |
||||
private RestTemplate restTemplate; |
||||
|
||||
// Ollama API地址
|
||||
private static final String OLLAMA_EMBEDDING_URL = "http://127.0.0.1:11434/api/embeddings"; |
||||
private static final String OLLAMA_GENERATION_URL = "http://127.0.0.1:11434/api/chat"; |
||||
|
||||
// 使用的模型
|
||||
private static final String EMBEDDING_MODEL = "nomic-embed-text:v1.5"; |
||||
private static final String GENERATION_MODEL = "qwen3:4b"; |
||||
|
||||
// 是否使用假数据
|
||||
@Value("${mock.enabled:false}") |
||||
private boolean mockEnabled; |
||||
|
||||
// 假数据答案
|
||||
@Value("${mock.qa.answer:这是默认的模拟答案}") |
||||
private String mockAnswer; |
||||
|
||||
// 假数据来源
|
||||
@Value("${mock.qa.sources:模拟来源1,模拟来源2}") |
||||
private String mockSources; |
||||
|
||||
/** |
||||
* 检查是否使用假数据模式 |
||||
* @return 是否使用假数据 |
||||
*/ |
||||
public boolean isMockEnabled() { |
||||
return mockEnabled; |
||||
} |
||||
|
||||
// 添加连接测试方法
|
||||
private boolean isOllamaReachable() { |
||||
try (Socket socket = new Socket()) { |
||||
socket.connect(new InetSocketAddress("127.0.0.1", 11434), 5000); // 5秒超时
|
||||
return true; |
||||
} catch (Exception e) { |
||||
System.err.println("无法通过Socket连接到Ollama服务: " + e.getMessage()); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
// 添加HTTP连接测试方法
|
||||
private boolean isOllamaHttpReachable() { |
||||
try { |
||||
ResponseEntity<String> response = restTemplate.getForEntity("http://127.0.0.1:11434/api/tags", String.class); |
||||
return response.getStatusCode().is2xxSuccessful(); |
||||
} catch (Exception e) { |
||||
System.err.println("无法通过HTTP连接到Ollama服务: " + e.getMessage()); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 获取文本的向量表示结果 |
||||
* |
||||
* @param text 输入文本 |
||||
* @return 向量表示结果 |
||||
*/ |
||||
public EmbeddingResponse getEmbedding(String text) { |
||||
// 如果启用假数据模式，返回随机向量
|
||||
if (mockEnabled) { |
||||
System.out.println("使用假数据模式，返回随机向量"); |
||||
return generateRandomEmbedding(); |
||||
} |
||||
|
||||
System.out.println("开始获取文本嵌入向量，输入文本长度: " + (text != null ? text.length() : 0)); |
||||
|
||||
// 验证输入文本长度是否符合API要求
|
||||
if (text == null || text.isEmpty()) { |
||||
System.err.println("输入文本为空"); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0); |
||||
} |
||||
|
||||
// 如果文本长度超过8192个字符，则截取前8192个字符
|
||||
if (text.length() > 8192) { |
||||
System.out.println("输入文本长度超过8192字符，已自动截取前8192个字符"); |
||||
text = text.substring(0, 8192); |
||||
} |
||||
|
||||
// 检查Ollama服务是否可达
|
||||
System.out.println("检查Ollama服务是否可达..."); |
||||
if (!isOllamaReachable()) { |
||||
System.err.println("Socket连接测试失败"); |
||||
if (!isOllamaHttpReachable()) { |
||||
System.err.println("HTTP连接测试也失败"); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "无法连接到Ollama服务，请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。"); |
||||
} else { |
||||
System.out.println("HTTP连接测试成功，但Socket连接失败，可能存在网络配置问题"); |
||||
} |
||||
} else { |
||||
System.out.println("Socket连接测试成功"); |
||||
} |
||||
|
||||
try { |
||||
// 构造请求体
|
||||
Map<String, String> requestBody = new HashMap<>(); |
||||
requestBody.put("model", EMBEDDING_MODEL); |
||||
requestBody.put("prompt", text); |
||||
|
||||
// 设置请求头
|
||||
HttpHeaders headers = new HttpHeaders(); |
||||
headers.setContentType(MediaType.APPLICATION_JSON); |
||||
|
||||
// 创建HTTP实体
|
||||
HttpEntity<Map<String, String>> requestEntity = new HttpEntity<>(requestBody, headers); |
||||
|
||||
// 发送POST请求到Ollama API
|
||||
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_EMBEDDING_URL, requestEntity, String.class); |
||||
|
||||
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { |
||||
// 解析响应
|
||||
ObjectMapper objectMapper = new ObjectMapper(); |
||||
JsonNode rootNode = objectMapper.readTree(response.getBody()); |
||||
|
||||
// 检查是否有错误信息
|
||||
JsonNode errorNode = rootNode.get("error"); |
||||
if (errorNode != null) { |
||||
String errorMessage = errorNode.asText(); |
||||
System.err.println("Ollama API返回错误: " + errorMessage); |
||||
// 特别处理模型未找到的情况
|
||||
if (errorMessage.contains("not found")) { |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到，请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型"); |
||||
} |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "Ollama API错误: " + errorMessage); |
||||
} |
||||
|
||||
// 尝试获取"embeddings"字段（某些Ollama版本）
|
||||
JsonNode embeddingsNode = rootNode.get("embeddings"); |
||||
if (embeddingsNode != null && embeddingsNode.isArray()) { |
||||
List<Float> embedding = new ArrayList<>(); |
||||
for (JsonNode node : embeddingsNode) { |
||||
embedding.add((float) node.asDouble()); |
||||
} |
||||
|
||||
System.out.println("成功获取嵌入向量，维度: " + embedding.size()); |
||||
return new EmbeddingResponse(true, embedding, embedding.size()); |
||||
} |
||||
|
||||
// 尝试获取"embedding"字段（其他Ollama版本）
|
||||
JsonNode embeddingNode = rootNode.get("embedding"); |
||||
if (embeddingNode != null && embeddingNode.isArray()) { |
||||
List<Float> embedding = new ArrayList<>(); |
||||
for (JsonNode node : embeddingNode) { |
||||
embedding.add((float) node.asDouble()); |
||||
} |
||||
|
||||
System.out.println("成功获取嵌入向量，维度: " + embedding.size()); |
||||
return new EmbeddingResponse(true, embedding, embedding.size()); |
||||
} |
||||
|
||||
// 如果两种方式都失败，记录完整的响应内容用于调试
|
||||
System.err.println("Ollama API返回的嵌入向量格式不正确，完整响应: " + rootNode.toString()); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "Ollama API返回的嵌入向量格式不正确"); |
||||
} else { |
||||
System.err.println("Ollama API调用失败，状态码: " + response.getStatusCode()); |
||||
// 特别处理404错误，提示用户需要拉取模型
|
||||
if (response.getStatusCode().value() == 404) { |
||||
System.err.println("模型未找到，请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型"); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到，请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型"); |
||||
} |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0); |
||||
} |
||||
} catch (ResourceAccessException e) { |
||||
// 处理连接被拒绝的异常
|
||||
if (e.getCause() instanceof java.net.ConnectException) { |
||||
System.err.println("无法连接到Ollama服务: " + e.getMessage()); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "无法连接到Ollama服务，请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。"); |
||||
} |
||||
System.err.println("调用Ollama嵌入API时发生网络异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "调用嵌入API时发生网络异常: " + e.getMessage()); |
||||
} catch (org.springframework.web.client.HttpClientErrorException.NotFound e) { |
||||
System.err.println("模型未找到: " + e.getMessage()); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到，请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型"); |
||||
} catch (Exception e) { |
||||
System.err.println("调用Ollama嵌入API时发生异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
return new EmbeddingResponse(false, new ArrayList<>(), 0, "调用嵌入API时发生异常: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 生成随机向量作为后备方案 |
||||
* |
||||
* @return 随机向量 |
||||
*/ |
||||
private EmbeddingResponse generateRandomEmbedding() { |
||||
List<Float> randomEmbedding = new ArrayList<>(); |
||||
Random random = new Random(); |
||||
for (int i = 0; i < 100; i++) { |
||||
randomEmbedding.add(random.nextFloat() * 2 - 1); // 生成-1到1之间的随机数
|
||||
} |
||||
System.out.println("Generated Random Embedding, Size: " + randomEmbedding.size()); |
||||
return new EmbeddingResponse(true, randomEmbedding, 100); |
||||
} |
||||
|
||||
/** |
||||
* 生成问题的答案（支持多轮对话） |
||||
* |
||||
* @param messages 对话历史消息列表 |
||||
* @return 生成的答案 |
||||
*/ |
||||
public String generateAnswer(List<Map<String, String>> messages) { |
||||
// 如果启用假数据模式，返回预设答案
|
||||
if (mockEnabled) { |
||||
System.out.println("使用假数据模式，返回预设答案"); |
||||
return mockAnswer; |
||||
} |
||||
|
||||
System.out.println("开始生成答案，消息数量: " + (messages != null ? messages.size() : 0)); |
||||
|
||||
// 验证消息列表
|
||||
if (messages == null || messages.isEmpty()) { |
||||
System.err.println("消息列表为空"); |
||||
return "抱歉，我无法回答这个问题。请稍后再试。"; |
||||
} |
||||
|
||||
// 检查Ollama服务是否可达
|
||||
System.out.println("检查Ollama服务是否可达..."); |
||||
if (!isOllamaReachable()) { |
||||
System.err.println("Socket连接测试失败"); |
||||
if (!isOllamaHttpReachable()) { |
||||
System.err.println("HTTP连接测试也失败"); |
||||
return "系统遇到错误: 无法连接到Ollama服务，请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。"; |
||||
} else { |
||||
System.out.println("HTTP连接测试成功，但Socket连接失败，可能存在网络配置问题"); |
||||
} |
||||
} else { |
||||
System.out.println("Socket连接测试成功"); |
||||
} |
||||
|
||||
// 添加重试机制
|
||||
int maxRetries = 5; // 增加重试次数
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) { |
||||
try { |
||||
// 构造请求体
|
||||
Map<String, Object> requestBody = new HashMap<>(); |
||||
requestBody.put("model", GENERATION_MODEL); |
||||
requestBody.put("messages", messages); |
||||
requestBody.put("stream", false); |
||||
requestBody.put("options", Map.of("num_predict", 2048)); // 限制生成长度以减少资源消耗
|
||||
|
||||
// 设置请求头
|
||||
HttpHeaders headers = new HttpHeaders(); |
||||
headers.setContentType(MediaType.APPLICATION_JSON); |
||||
|
||||
// 创建HTTP实体
|
||||
HttpEntity<Map<String, Object>> requestEntity = new HttpEntity<>(requestBody, headers); |
||||
|
||||
// 发送POST请求到Ollama API
|
||||
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_GENERATION_URL, requestEntity, String.class); |
||||
|
||||
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { |
||||
System.out.println("Ollama API 响应: " + response.getBody()); |
||||
|
||||
// 解析响应
|
||||
ObjectMapper objectMapper = new ObjectMapper(); |
||||
JsonNode rootNode = objectMapper.readTree(response.getBody()); |
||||
|
||||
// 检查是否有错误信息
|
||||
JsonNode errorNode = rootNode.get("error"); |
||||
if (errorNode != null) { |
||||
String errorMessage = errorNode.asText(); |
||||
System.err.println("Ollama API返回错误: " + errorMessage); |
||||
// 特别处理模型未找到的情况
|
||||
if (errorMessage.contains("not found")) { |
||||
return "系统遇到错误: 模型 '" + GENERATION_MODEL + "' 未找到，请联系管理员确认模型是否已正确安装"; |
||||
} |
||||
return "系统遇到错误: " + errorMessage + "。请稍后再试。"; |
||||
} |
||||
|
||||
// 检查done_reason是否为load（模型正在加载）
|
||||
JsonNode doneReasonNode = rootNode.get("done_reason"); |
||||
if (doneReasonNode != null) { |
||||
String doneReason = doneReasonNode.asText(); |
||||
// 正确处理"load"状态 - 忽略它并继续等待结果
|
||||
if ("load".equals(doneReason)) { |
||||
System.out.println("第 " + attempt + " 次尝试：模型正在加载中，请稍后再试"); |
||||
if (attempt < maxRetries) { |
||||
// 等待时间逐渐增加
|
||||
long waitTime = 5000 * attempt; // 5秒, 10秒, 15秒, 20秒...
|
||||
System.out.println("等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} else { |
||||
return "AI模型正在加载中，请稍等2-3分钟后再试。如果问题持续存在，可能需要检查系统资源配置。"; |
||||
} |
||||
} |
||||
// 只有当done_reason为stop时才表示真正结束
|
||||
else if (!"stop".equals(doneReason)) { |
||||
System.out.println("第 " + attempt + " 次尝试：模型返回未知的done_reason: " + doneReason); |
||||
if (attempt < maxRetries) { |
||||
long waitTime = 3000 * attempt; |
||||
System.out.println("等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} |
||||
} |
||||
} |
||||
|
||||
// 首先尝试获取"message"字段（适用于某些Ollama版本）
|
||||
JsonNode messageNode = rootNode.get("message"); |
||||
if (messageNode != null) { |
||||
JsonNode contentNode = messageNode.get("content"); |
||||
if (contentNode != null) { |
||||
String content = contentNode.asText(); |
||||
System.out.println("从 'message.content' 获取回答内容，长度: " + (content != null ? content.length() : 0)); |
||||
if (content != null && !content.isEmpty()) { |
||||
return content; |
||||
} |
||||
} |
||||
} |
||||
|
||||
// 如果没有"message"字段，尝试直接获取"response"字段（适用于其他Ollama版本）
|
||||
JsonNode responseNode = rootNode.get("response"); |
||||
if (responseNode != null) { |
||||
String content = responseNode.asText(); |
||||
System.out.println("从 'response' 获取回答内容，长度: " + (content != null ? content.length() : 0)); |
||||
if (content != null && !content.isEmpty()) { |
||||
return content; |
||||
} |
||||
} |
||||
|
||||
// 尝试获取done字段为false时的文本（流式响应但stream=false的情况）
|
||||
JsonNode doneNode = rootNode.get("done"); |
||||
if (doneNode != null && !doneNode.asBoolean()) { |
||||
JsonNode textNode = rootNode.get("response"); |
||||
if (textNode != null) { |
||||
String content = textNode.asText(); |
||||
System.out.println("从流式响应获取回答内容，长度: " + (content != null ? content.length() : 0)); |
||||
if (content != null && !content.isEmpty()) { |
||||
return content; |
||||
} |
||||
} |
||||
} |
||||
|
||||
// 如果以上方式都失败，检查是否有其他有用信息
|
||||
JsonNode modelNode = rootNode.get("model"); |
||||
JsonNode createdAtNode = rootNode.get("created_at"); |
||||
if (modelNode != null && createdAtNode != null) { |
||||
String model = modelNode.asText(); |
||||
String createdAt = createdAtNode.asText(); |
||||
System.out.println("模型信息 - 模型: " + model + ", 创建时间: " + createdAt); |
||||
// 如果只有模型信息但没有内容，说明可能生成了空响应
|
||||
if (rootNode.size() <= 3) { // 只有model, created_at, done等基本信息
|
||||
if (attempt < maxRetries) { |
||||
// 等待一段时间再重试
|
||||
long waitTime = 3000 * attempt; // 3秒, 6秒, 9秒...
|
||||
System.out.println("模型响应不完整，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} |
||||
return "AI模型未能生成相关回答，请尝试重新提问或联系管理员。"; |
||||
} |
||||
} |
||||
|
||||
// 如果两种方式都失败，记录完整的响应内容用于调试
|
||||
System.err.println("Ollama API返回的结果格式不正确，完整响应: " + rootNode.toString()); |
||||
return "抱歉，我无法回答这个问题。请稍后再试。"; |
||||
} else { |
||||
System.err.println("Ollama API调用失败，状态码: " + response.getStatusCode()); |
||||
if (attempt < maxRetries) { |
||||
// 等待一段时间再重试
|
||||
long waitTime = 3000 * attempt; |
||||
System.out.println("API调用失败，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} |
||||
return "抱歉，我无法回答这个问题。请稍后再试。"; |
||||
} |
||||
} catch (ResourceAccessException e) { |
||||
// 处理连接被拒绝的异常
|
||||
if (e.getCause() instanceof java.net.ConnectException) { |
||||
System.err.println("无法连接到Ollama服务: " + e.getMessage()); |
||||
return "系统遇到错误: 无法连接到Ollama服务，请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。"; |
||||
} |
||||
System.err.println("调用Ollama API时发生网络异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
if (attempt < maxRetries) { |
||||
try { |
||||
long waitTime = 5000 * attempt; |
||||
System.out.println("发生网络异常，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
} catch (InterruptedException ie) { |
||||
Thread.currentThread().interrupt(); |
||||
return "抱歉，我无法回答这个问题。系统遇到了一个意外错误，请稍后再试。"; |
||||
} |
||||
continue; |
||||
} |
||||
return "抱歉，我无法回答这个问题。系统遇到了网络连接错误，请稍后再试。"; |
||||
} catch (Exception e) { |
||||
System.err.println("调用Ollama API时发生异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
if (attempt < maxRetries) { |
||||
try { |
||||
long waitTime = 5000 * attempt; |
||||
System.out.println("发生异常，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
} catch (InterruptedException ie) { |
||||
Thread.currentThread().interrupt(); |
||||
return "抱歉，我无法回答这个问题。系统遇到了一个意外错误，请稍后再试。"; |
||||
} |
||||
continue; |
||||
} |
||||
return "抱歉，我无法回答这个问题。系统遇到了一个意外错误，请稍后再试。"; |
||||
} |
||||
} |
||||
|
||||
return "抱歉，系统尝试多次仍无法获取答案，请稍后再试。"; |
||||
} |
||||
|
||||
|
||||
|
||||
/** |
||||
* 自动生成知识条目 |
||||
* 调用大语言模型对输入内容进行分段和总结，生成临时知识条目 |
||||
* |
||||
* @param content 原始内容 |
||||
* @param source 知识来源 |
||||
* @return 分段后的内容列表 |
||||
*/ |
||||
public List<String> generateKnowledgeEntries(String content, String source) { |
||||
// 如果启用假数据模式，返回模拟分段
|
||||
if (mockEnabled) { |
||||
System.out.println("使用假数据模式，返回模拟分段"); |
||||
List<String> mockSegments = new ArrayList<>(); |
||||
mockSegments.add("这是模拟的知识条目1。在真实环境中，这将是由AI模型生成的内容。"); |
||||
mockSegments.add("这是模拟的知识条目2。它代表了原始内容的另一个片段。"); |
||||
mockSegments.add("这是模拟的知识条目3。用于演示知识库的多条目结构。"); |
||||
return mockSegments; |
||||
} |
||||
|
||||
// 如果内容为空，直接返回默认分段
|
||||
if (content == null || content.trim().isEmpty()) { |
||||
List<String> defaultSegments = new ArrayList<>(); |
||||
defaultSegments.add("待处理内容"); |
||||
return defaultSegments; |
||||
} |
||||
|
||||
// 限制内容长度，避免API调用失败
|
||||
String processedContent = content; |
||||
if (content == null || content.trim().isEmpty()) { |
||||
System.out.println("输入内容为空，使用默认内容"); |
||||
processedContent = "未提供有效内容"; |
||||
} else if (content.length() > 8000) { |
||||
System.out.println("输入内容长度超过8000字符，已自动截取前8000个字符"); |
||||
processedContent = content.substring(0, 8000); |
||||
} else { |
||||
System.out.println("使用完整内容，长度: " + content.length()); |
||||
} |
||||
|
||||
// 添加重试机制
|
||||
int maxRetries = 3; |
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) { |
||||
try { |
||||
// 构造提示词，要求模型将内容分段并总结
|
||||
String prompt = "请将以下内容分段并总结，根据知识来源进行分段，内容不能太少，没有字数要求。" + |
||||
"请按照以下JSON格式返回结果，不要包含其他内容：\n" + |
||||
"[\n" + |
||||
" \"分段内容1\",\n" + |
||||
" \"分段内容2\",\n" + |
||||
" \"分段内容3\"\n" + |
||||
"]\n\n" + |
||||
"知识来源：" + source + "\n\n" + |
||||
"内容如下：\n" + processedContent; |
||||
|
||||
System.out.println("构造提示词完成，提示词长度: " + prompt.length()); |
||||
System.out.println("使用的模型: " + GENERATION_MODEL); |
||||
|
||||
// 构造请求体
|
||||
Map<String, Object> requestBody = new HashMap<>(); |
||||
requestBody.put("model", GENERATION_MODEL); |
||||
requestBody.put("prompt", prompt); |
||||
requestBody.put("stream", false); |
||||
requestBody.put("temperature", 0.7); |
||||
|
||||
// 设置请求头
|
||||
HttpHeaders headers = new HttpHeaders(); |
||||
headers.setContentType(MediaType.APPLICATION_JSON); |
||||
|
||||
// 创建HTTP实体
|
||||
HttpEntity<Map<String, Object>> requestEntity = new HttpEntity<>(requestBody, headers); |
||||
|
||||
// 发送POST请求到Ollama API
|
||||
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_GENERATION_URL, requestEntity, String.class); |
||||
|
||||
// 解析响应
|
||||
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { |
||||
// 解析响应
|
||||
ObjectMapper objectMapper = new ObjectMapper(); |
||||
JsonNode rootNode = objectMapper.readTree(response.getBody()); |
||||
|
||||
// 检查是否有错误信息
|
||||
JsonNode errorNode = rootNode.get("error"); |
||||
if (errorNode != null) { |
||||
String errorMessage = errorNode.asText(); |
||||
System.err.println("Ollama API返回错误: " + errorMessage); |
||||
// 特别处理模型未找到的情况
|
||||
if (errorMessage.contains("not found")) { |
||||
throw new RuntimeException("模型 '" + GENERATION_MODEL + "' 未找到，请联系管理员确认模型是否已正确安装"); |
||||
} |
||||
throw new RuntimeException("Ollama API错误: " + errorMessage); |
||||
} |
||||
|
||||
// 检查done_reason是否为load（模型正在加载）
|
||||
JsonNode doneReasonNode = rootNode.get("done_reason"); |
||||
if (doneReasonNode != null) { |
||||
String doneReason = doneReasonNode.asText(); |
||||
// 正确处理"load"状态 - 忽略它并继续等待结果
|
||||
if ("load".equals(doneReason)) { |
||||
System.out.println("第 " + attempt + " 次尝试：模型正在加载中，请稍后再试"); |
||||
if (attempt < maxRetries) { |
||||
// 等待时间逐渐增加
|
||||
long waitTime = 5000 * attempt; |
||||
System.out.println("等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} else { |
||||
throw new RuntimeException("AI模型正在加载中，请稍等2-3分钟后再试。如果问题持续存在，可能需要检查系统资源配置。"); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// 尝试获取"response"字段
|
||||
String responseText = ""; |
||||
JsonNode responseNode = rootNode.get("response"); |
||||
if (responseNode != null) { |
||||
responseText = responseNode.asText(); |
||||
} else { |
||||
// 如果没有"response"字段，尝试使用整个响应体
|
||||
responseText = rootNode.toString(); |
||||
} |
||||
|
||||
// 打印AI生成的原始结果，便于查看格式是否正确
|
||||
System.out.println("=== AI生成的原始切片结果 ==="); |
||||
System.out.println(responseText); |
||||
System.out.println("=== 结束 ==="); |
||||
|
||||
// 检查返回结果是否为空
|
||||
if (responseText.isEmpty()) { |
||||
return createFallbackSegments(processedContent); |
||||
} |
||||
|
||||
// 尝试解析JSON格式的响应
|
||||
try { |
||||
List<String> segments = objectMapper.readValue(responseText, new TypeReference<List<String>>() { |
||||
}); |
||||
return segments; |
||||
} catch (Exception jsonException) { |
||||
// 如果JSON解析失败，尝试从文本中提取内容
|
||||
System.out.println("JSON解析失败，尝试从文本中提取内容"); |
||||
return extractSegmentsFromNonJsonText(responseText); |
||||
} |
||||
} else { |
||||
System.err.println("Ollama API调用失败，状态码: " + response.getStatusCode()); |
||||
if (attempt < maxRetries) { |
||||
long waitTime = 3000 * attempt; |
||||
System.out.println("API调用失败，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
continue; |
||||
} |
||||
return createFallbackSegments(processedContent); |
||||
} |
||||
} catch (Exception e) { |
||||
System.err.println("调用Ollama API时发生异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
if (attempt < maxRetries) { |
||||
try { |
||||
long waitTime = 5000 * attempt; |
||||
System.out.println("发生异常，等待 " + (waitTime/1000) + " 秒后重试..."); |
||||
Thread.sleep(waitTime); |
||||
} catch (InterruptedException ie) { |
||||
Thread.currentThread().interrupt(); |
||||
List<String> errorSegments = new ArrayList<>(); |
||||
errorSegments.add("处理内容时发生错误: " + e.getMessage()); |
||||
return errorSegments; |
||||
} |
||||
continue; |
||||
} |
||||
List<String> errorSegments = new ArrayList<>(); |
||||
errorSegments.add("处理内容时发生错误: " + e.getMessage()); |
||||
return errorSegments; |
||||
} |
||||
} |
||||
|
||||
List<String> errorSegments = new ArrayList<>(); |
||||
errorSegments.add("处理内容时发生未知错误，请稍后再试"); |
||||
return errorSegments; |
||||
} |
||||
|
||||
/** |
||||
* 从文本中提取分段内容 |
||||
* 当JSON解析失败时使用此方法 |
||||
* |
||||
* @param text AI返回的文本 |
||||
* @return 分段内容列表 |
||||
*/ |
||||
private List<String> extractSegmentsFromNonJsonText(String text) { |
||||
List<String> segments = new ArrayList<>(); |
||||
|
||||
if (text == null || text.trim().isEmpty()) { |
||||
segments.add("内容待处理"); |
||||
return segments; |
||||
} |
||||
|
||||
try { |
||||
// 移除可能的代码块标记
|
||||
String cleanText = text.replace("```json", "").replace("```", "").trim(); |
||||
|
||||
// 尝试直接解析JSON数组
|
||||
ObjectMapper objectMapper = new ObjectMapper(); |
||||
try { |
||||
List<String> jsonSegments = objectMapper.readValue(cleanText, new TypeReference<List<String>>() { |
||||
}); |
||||
if (!jsonSegments.isEmpty()) { |
||||
System.out.println("成功解析JSON数组，获得" + jsonSegments.size() + "个分段"); |
||||
return jsonSegments; |
||||
} |
||||
} catch (Exception e) { |
||||
System.out.println("直接JSON解析失败，继续尝试其他方法: " + e.getMessage()); |
||||
} |
||||
|
||||
// 尝试查找并解析包含在方括号中的JSON数组
|
||||
int startIndex = cleanText.indexOf('['); |
||||
int endIndex = cleanText.lastIndexOf(']'); |
||||
if (startIndex != -1 && endIndex != -1 && endIndex > startIndex) { |
||||
String jsonArrayStr = cleanText.substring(startIndex, endIndex + 1); |
||||
try { |
||||
List<String> jsonSegments = objectMapper.readValue(jsonArrayStr, new TypeReference<List<String>>() { |
||||
}); |
||||
if (!jsonSegments.isEmpty()) { |
||||
System.out.println("成功解析提取的JSON数组，获得" + jsonSegments.size() + "个分段"); |
||||
return jsonSegments; |
||||
} |
||||
} catch (Exception e) { |
||||
System.out.println("提取的JSON数组解析失败: " + e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
// 按引号分割提取内容
|
||||
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\"([^\"]+)\""); |
||||
java.util.regex.Matcher matcher = pattern.matcher(cleanText); |
||||
|
||||
while (matcher.find()) { |
||||
String segment = matcher.group(1).trim(); |
||||
if (!segment.isEmpty() && segment.length() > 10) { |
||||
segments.add(segment); |
||||
} |
||||
} |
||||
|
||||
// 如果没有通过引号提取到内容，尝试按行分割
|
||||
if (segments.isEmpty()) { |
||||
String[] lines = cleanText.split("\n"); |
||||
for (String line : lines) { |
||||
String cleanLine = line.trim().replaceAll("^[0-9]+\\.\\s*", "").replaceAll("\"", ""); |
||||
if (!cleanLine.isEmpty() && cleanLine.length() > 10) { |
||||
segments.add(cleanLine); |
||||
} |
||||
} |
||||
} |
||||
|
||||
} catch (Exception e) { |
||||
System.err.println("从非JSON文本中提取分段内容时发生异常: " + e.getMessage()); |
||||
e.printStackTrace(); |
||||
} |
||||
|
||||
// 如果仍然没有内容，添加原始文本作为单个分段
|
||||
if (segments.isEmpty()) { |
||||
String finalText = text.length() > 200 ? text.substring(0, 200) + "..." : text; |
||||
segments.add(finalText); |
||||
System.out.println("使用原始文本作为单个分段: " + finalText); |
||||
} |
||||
|
||||
System.out.println("从非JSON文本中提取到" + segments.size() + "个分段"); |
||||
return segments; |
||||
} |
||||
|
||||
/** |
||||
* 创建备用分段方案 |
||||
* 当AI分段失败时使用此方法 |
||||
* |
||||
* @param content 原始内容 |
||||
* @return 备用分段列表 |
||||
*/ |
||||
private List<String> createFallbackSegments(String content) { |
||||
System.out.println("使用备用分段方案"); |
||||
List<String> segments = new ArrayList<>(); |
||||
|
||||
if (content == null || content.trim().isEmpty()) { |
||||
segments.add("待处理内容"); |
||||
return segments; |
||||
} |
||||
|
||||
String trimmedContent = content.trim(); |
||||
|
||||
// 简单按句号分割
|
||||
String[] sentences = trimmedContent.split("[。.!?！？]"); |
||||
for (String sentence : sentences) { |
||||
String cleanSentence = sentence.trim(); |
||||
if (!cleanSentence.isEmpty() && cleanSentence.length() > 10) { |
||||
segments.add(cleanSentence); |
||||
} |
||||
} |
||||
|
||||
// 如果没有按句号分割出内容，尝试按段落分割
|
||||
if (segments.isEmpty()) { |
||||
String[] paragraphs = trimmedContent.split("\n\n"); |
||||
for (String paragraph : paragraphs) { |
||||
String cleanParagraph = paragraph.trim(); |
||||
if (!cleanParagraph.isEmpty() && cleanParagraph.length() > 20) { |
||||
segments.add(cleanParagraph); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// 如果仍然没有内容，创建一个默认分段
|
||||
if (segments.isEmpty()) { |
||||
// 创建一个包含前200个字符的分段
|
||||
String shortContent = trimmedContent.substring(0, Math.min(200, trimmedContent.length())); |
||||
segments.add(shortContent + (trimmedContent.length() > 200 ? "..." : "")); |
||||
} |
||||
|
||||
System.out.println("备用方案创建了" + segments.size() + "个分段"); |
||||
return segments; |
||||
} |
||||
} |
||||
@ -0,0 +1,113 @@ |
||||
// service/EmbeddingService.java
|
||||
package com.alops.chat.service; |
||||
|
||||
import com.alops.chat.DTO.EmbeddingResponse; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
/** |
||||
* 向量嵌入服务类 |
||||
* 提供文本向量化功能 |
||||
*/ |
||||
@Service |
||||
public class EmbeddingService { |
||||
|
||||
@Autowired |
||||
private AItokenService AItokenService; |
||||
|
||||
/** |
||||
* 获取文本的向量表示 |
||||
* @param text 输入文本 |
||||
* @return 浮点数组形式的向量 |
||||
*/ |
||||
public float[] getEmbedding(String text) { |
||||
// 获取嵌入向量
|
||||
EmbeddingResponse response = AItokenService.getEmbedding(text); |
||||
|
||||
// 检查是否有错误信息
|
||||
if (response.getErrorMessage() != null && !response.getErrorMessage().isEmpty()) { |
||||
System.err.println("获取嵌入向量时出错: " + response.getErrorMessage()); |
||||
// 返回空数组而不是抛出异常
|
||||
return new float[0]; |
||||
} |
||||
|
||||
// 如果成功获取向量，则转换为浮点数组
|
||||
if (response.isSuccess()) { |
||||
float[] embedding = new float[response.getVector().size()]; |
||||
for (int i = 0; i < response.getVector().size(); i++) { |
||||
embedding[i] = response.getVector().get(i); |
||||
} |
||||
return embedding; |
||||
} |
||||
|
||||
// 如果获取失败，返回空数组
|
||||
return new float[0]; |
||||
} |
||||
|
||||
/** |
||||
* 将浮点数组转换为字节数组 |
||||
* @param floats 浮点数组 |
||||
* @return 字节数组 |
||||
*/ |
||||
public static byte[] floatArrayToByteArray(float[] floats) { |
||||
java.nio.ByteBuffer buffer = java.nio.ByteBuffer.allocate(floats.length * 4); |
||||
for (float f : floats) { |
||||
buffer.putFloat(f); |
||||
} |
||||
return buffer.array(); |
||||
} |
||||
|
||||
/** |
||||
* 将字节数组转换为浮点数组 |
||||
* @param bytes 字节数组 |
||||
* @return 浮点数组 |
||||
*/ |
||||
public static float[] byteArrayToFloatArray(byte[] bytes) { |
||||
if (bytes == null || bytes.length == 0) { |
||||
return new float[0]; |
||||
} |
||||
|
||||
java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(bytes); |
||||
float[] floats = new float[bytes.length / 4]; |
||||
for (int i = 0; i < floats.length; i++) { |
||||
floats[i] = buffer.getFloat(); |
||||
} |
||||
return floats; |
||||
} |
||||
|
||||
/** |
||||
* 计算两个向量之间的欧几里得距离 |
||||
* @param a 向量A |
||||
* @param b 向量B |
||||
* @return 欧几里得距离 |
||||
*/ |
||||
public static double calculateEuclideanDistance(float[] a, float[] b) { |
||||
if (a.length != b.length) { |
||||
// 当向量维度不匹配时，返回一个较大的距离值而不是抛出异常
|
||||
// 这样可以确保程序继续运行，同时将不匹配的向量排在后面
|
||||
return Double.MAX_VALUE; |
||||
} |
||||
|
||||
double sum = 0; |
||||
for (int i = 0; i < a.length; i++) { |
||||
double diff = a[i] - b[i]; |
||||
sum += diff * diff; |
||||
} |
||||
return Math.sqrt(sum); |
||||
} |
||||
|
||||
/** |
||||
* 检查Ollama服务是否可用 |
||||
* @return 服务是否可用 |
||||
*/ |
||||
public boolean isOllamaServiceAvailable() { |
||||
try { |
||||
// 尝试获取一个简单的嵌入向量来检查服务是否可用
|
||||
float[] testEmbedding = getEmbedding("test"); |
||||
return testEmbedding.length > 0; |
||||
} catch (Exception e) { |
||||
System.err.println("检查Ollama服务可用性时出错: " + e.getMessage()); |
||||
return false; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,469 @@ |
||||
// service/KnowledgeService.java
|
||||
package com.alops.chat.service; |
||||
|
||||
|
||||
import com.alops.chat.entity.KnowledgeEntity; |
||||
import com.alops.chat.repository.KnowledgeRepository; |
||||
import com.alops.chat.DTO.AnswerResponse; |
||||
import javax.persistence.EntityManager; |
||||
import javax.persistence.PersistenceContext; |
||||
import org.apache.poi.xwpf.usermodel.XWPFDocument; |
||||
import org.apache.poi.xwpf.usermodel.XWPFParagraph; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.beans.factory.annotation.Value; |
||||
import org.springframework.stereotype.Service; |
||||
import org.springframework.transaction.annotation.Transactional; |
||||
import org.springframework.web.multipart.MultipartFile; |
||||
|
||||
import java.io.BufferedReader; |
||||
import java.io.IOException; |
||||
import java.io.InputStreamReader; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
import java.nio.file.StandardCopyOption; |
||||
import java.time.LocalDateTime; |
||||
import java.time.format.DateTimeFormatter; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* 知识服务类 |
||||
* 提供知识库管理功能，包括添加、删除、更新知识条目，以及查找相似知识等功能 |
||||
*/ |
||||
@Service |
||||
public class KnowledgeService { |
||||
|
||||
@Autowired |
||||
private KnowledgeRepository knowledgeRepository; |
||||
|
||||
@Autowired |
||||
private EmbeddingService embeddingService; |
||||
|
||||
@Autowired |
||||
private AItokenService AItokenService; |
||||
|
||||
@Autowired |
||||
private TextSplitter textSplitter; |
||||
|
||||
// 是否使用假数据
|
||||
@Value("${mock.knowledge.enabled:false}") |
||||
private boolean mockKnowledgeEnabled; |
||||
|
||||
@PersistenceContext |
||||
private EntityManager entityManager; |
||||
|
||||
/** |
||||
* 添加知识到知识库 |
||||
* 直接分割内容并生成正式条目 |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @return 知识条目ID列表 |
||||
*/ |
||||
@Transactional |
||||
public List<Long> addKnowledge(String content, String source) { |
||||
return addKnowledgeWithFileId(content, source, null); |
||||
} |
||||
|
||||
/** |
||||
* 添加知识到知识库，包含文件ID |
||||
* 直接分割内容并生成正式条目 |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @param fileId 文件ID |
||||
* @return 知识条目ID列表 |
||||
*/ |
||||
@Transactional |
||||
public List<Long> addKnowledgeWithFileId(String content, String source, Long fileId) { |
||||
System.out.println("开始添加知识: " + content); |
||||
List<Long> knowledgeIds = new ArrayList<>(); |
||||
|
||||
// 如果使用假数据模式，添加模拟数据
|
||||
if (mockKnowledgeEnabled && (content == null || content.trim().isEmpty())) { |
||||
System.out.println("使用假数据模式，添加模拟知识条目"); |
||||
for (int i = 0; i < 3; i++) { |
||||
String mockContent = "模拟知识条目 " + (i + 1) + "。这是用于测试的假数据内容。"; |
||||
String mockSource = "模拟来源"; |
||||
Long id = addKnowledgeSegment(mockContent, mockSource, fileId != null ? fileId.intValue() : 0); |
||||
if (id != null) { |
||||
knowledgeIds.add(id); |
||||
} |
||||
} |
||||
return knowledgeIds; |
||||
} |
||||
|
||||
// 直接分割内容并生成正式条目
|
||||
if (content.length() > 200) { |
||||
System.out.println("内容长度超过200字符，自动切分并生成正式条目"); |
||||
List<String> segments = textSplitter.splitText(content); |
||||
for (int i = 0; i < segments.size(); i++) { |
||||
String segment = segments.get(i); |
||||
String segmentSource = source + " (part " + (i + 1) + "/" + segments.size() + ")"; |
||||
Long id = addKnowledgeSegment(segment, segmentSource, fileId != null ? fileId.intValue() : 0); |
||||
if (id != null) { |
||||
knowledgeIds.add(id); |
||||
} |
||||
} |
||||
} else { |
||||
// 直接添加知识
|
||||
Long id = addKnowledgeSegment(content, source, fileId != null ? fileId.intValue() : 0); |
||||
if (id != null) { |
||||
knowledgeIds.add(id); |
||||
} |
||||
} |
||||
|
||||
return knowledgeIds; |
||||
} |
||||
|
||||
/** |
||||
* 添加知识片段到知识库 |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @return 知识条目ID |
||||
*/ |
||||
@Transactional |
||||
public Long addKnowledgeSegment(String content, String source,int keyfileid) { |
||||
System.out.println("开始添加知识片段: " + content); |
||||
// 获取文本的向量表示
|
||||
float[] embedding = embeddingService.getEmbedding(content); |
||||
|
||||
System.out.println("接收到长度为 " + embedding.length + " 的向量"); |
||||
|
||||
if (embedding.length > 0) { |
||||
// 将浮点数向量转换为字节数组以便存储
|
||||
byte[] embeddingBytes = EmbeddingService.floatArrayToByteArray(embedding); |
||||
System.out.println("转换为长度为 " + embeddingBytes.length + " 的字节数组"); |
||||
|
||||
// 创建知识条目对象
|
||||
KnowledgeEntity entity = new KnowledgeEntity(content, embeddingBytes, source,keyfileid); |
||||
System.out.println("创建KnowledgeEntry对象"); |
||||
|
||||
// 保存到数据库
|
||||
KnowledgeEntity savedEntity = knowledgeRepository.save(entity); |
||||
System.out.println("保存KnowledgeEntry，ID: " + savedEntity.getId()); |
||||
|
||||
return savedEntity.getId(); |
||||
} else { |
||||
System.out.println("接收到空向量，跳过保存"); |
||||
return null; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 根据文件ID删除知识条目 |
||||
* @param fileId 文件ID |
||||
* @return 删除的条目数量 |
||||
*/ |
||||
@Transactional |
||||
public int deleteKnowledgeByFileId(Long fileId) { |
||||
return knowledgeRepository.deleteByKeyfileid(fileId.intValue()); |
||||
} |
||||
|
||||
/** |
||||
* 自动生成知识条目 |
||||
* 调用大语言模型对输入内容进行分段和总结，生成临时知识条目 |
||||
* |
||||
* @param content 原始内容 |
||||
* @param source 知识来源 |
||||
* @return 分段后的内容列表 |
||||
*/ |
||||
public List<String> generateKnowledgeEntries(String content, String source) { |
||||
return AItokenService.generateKnowledgeEntries(content, source); |
||||
} |
||||
|
||||
/** |
||||
* 获取所有正式条目 |
||||
* @return 正式条目列表 |
||||
*/ |
||||
public List<KnowledgeEntity> getPermanentEntries() { |
||||
// 如果使用假数据模式，返回模拟数据
|
||||
if (mockKnowledgeEnabled) { |
||||
System.out.println("使用假数据模式，返回模拟知识条目"); |
||||
List<KnowledgeEntity> mockEntries = new ArrayList<>(); |
||||
for (int i = 0; i < 5; i++) { |
||||
KnowledgeEntity mockEntry = new KnowledgeEntity(); |
||||
mockEntry.setId((long) (i + 1)); |
||||
mockEntry.setContent("模拟知识条目 " + (i + 1) + "。这是用于测试的假数据内容。"); |
||||
mockEntry.setSource("模拟来源"); |
||||
mockEntry.setCreatetime(LocalDateTime.now()); |
||||
mockEntry.setUpdatetime(LocalDateTime.now()); |
||||
mockEntries.add(mockEntry); |
||||
} |
||||
return mockEntries; |
||||
} |
||||
|
||||
return knowledgeRepository.findAll(); |
||||
} |
||||
|
||||
|
||||
|
||||
/** |
||||
* 删除指定ID的知识条目 |
||||
* @param id 知识条目ID |
||||
* @return 删除是否成功 |
||||
*/ |
||||
@Transactional |
||||
public boolean deleteKnowledge(Long id) { |
||||
if (knowledgeRepository.existsById(id)) { |
||||
knowledgeRepository.deleteById(id); |
||||
return true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* 提取问题中的关键词 |
||||
* @param question 用户问题 |
||||
* @return 关键词列表 |
||||
*/ |
||||
private List<String> extractKeywords(String question) { |
||||
List<String> keywords = new ArrayList<>(); |
||||
|
||||
if (question == null || question.trim().isEmpty()) { |
||||
return keywords; |
||||
} |
||||
|
||||
// 移除常见的疑问词和停用词
|
||||
String cleanedQuestion = question.replaceAll( |
||||
"(什么|如何|为什么|怎么|哪里|哪个|谁|何时|是否|怎么办|吗|呢|吧|啊|的|了|在|是|有|和|与|及|以及|或者|或)$", |
||||
"").trim(); |
||||
|
||||
// 按空格和常见标点符号分割
|
||||
String[] words = cleanedQuestion.split("[\\s\\p{Punct}]+"); |
||||
|
||||
// 过滤掉太短的词（少于2个字符）
|
||||
for (String word : words) { |
||||
if (word.length() >= 2) { |
||||
keywords.add(word); |
||||
} |
||||
} |
||||
|
||||
return keywords; |
||||
} |
||||
|
||||
/** |
||||
* 基于关键词和向量的混合检索 |
||||
* @param question 用户问题 |
||||
* @param limit 返回结果数量上限 |
||||
* @return 相似知识条目列表 |
||||
*/ |
||||
public List<KnowledgeEntity> findSimilarKnowledgeWithKeywordFilter(String question, int limit) { |
||||
// 如果使用假数据模式，返回模拟数据
|
||||
if (mockKnowledgeEnabled) { |
||||
System.out.println("使用假数据模式，返回模拟相似知识条目"); |
||||
List<KnowledgeEntity> mockEntries = new ArrayList<>(); |
||||
for (int i = 0; i < Math.min(3, limit); i++) { |
||||
KnowledgeEntity mockEntry = new KnowledgeEntity(); |
||||
mockEntry.setId((long) (i + 1)); |
||||
mockEntry.setContent("这是与问题 \"" + question + "\" 相关的模拟知识条目 " + (i + 1) + "。在真实环境中，这将是通过向量相似度检索得到的结果。"); |
||||
mockEntry.setSource("模拟来源 " + (i + 1)); |
||||
mockEntry.setCreatetime(LocalDateTime.now()); |
||||
mockEntry.setUpdatetime(LocalDateTime.now()); |
||||
mockEntries.add(mockEntry); |
||||
} |
||||
return mockEntries; |
||||
} |
||||
|
||||
// 1. 提取关键词
|
||||
List<String> keywords = extractKeywords(question); |
||||
System.out.println("提取到关键词: " + keywords); |
||||
|
||||
// 2. 如果有关键词，先进行关键词搜索
|
||||
List<KnowledgeEntity> keywordFilteredEntries = new ArrayList<>(); |
||||
if (!keywords.isEmpty()) { |
||||
// 使用前3个关键词进行搜索
|
||||
String keyword1 = keywords.size() > 0 ? keywords.get(0) : ""; |
||||
String keyword2 = keywords.size() > 1 ? keywords.get(1) : ""; |
||||
String keyword3 = keywords.size() > 2 ? keywords.get(2) : ""; |
||||
|
||||
keywordFilteredEntries = knowledgeRepository.findByMultipleKeywords(keyword1, keyword2, keyword3); |
||||
System.out.println("关键词搜索返回 " + keywordFilteredEntries.size() + " 条结果"); |
||||
} |
||||
|
||||
// 3. 如果关键词搜索没有结果，则使用所有知识条目
|
||||
List<KnowledgeEntity> candidateEntries = keywordFilteredEntries.isEmpty() ? |
||||
knowledgeRepository.findAll() : keywordFilteredEntries; |
||||
|
||||
// 4. 在候选条目中进行向量相似度检索
|
||||
return findSimilarKnowledgeFromCandidates(question, candidateEntries, limit); |
||||
} |
||||
|
||||
/** |
||||
* 从候选条目中查找与问题最相似的知识条目 |
||||
* @param question 用户问题 |
||||
* @param candidates 候选知识条目 |
||||
* @param limit 返回结果数量上限 |
||||
* @return 相似知识条目列表 |
||||
*/ |
||||
private List<KnowledgeEntity> findSimilarKnowledgeFromCandidates(String question, List<KnowledgeEntity> candidates, int limit) { |
||||
// 获取问题的向量表示
|
||||
float[] questionEmbedding = embeddingService.getEmbedding(question); |
||||
|
||||
// 如果无法生成向量，则返回空列表
|
||||
if (questionEmbedding.length == 0) { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
// 根据欧几里得距离对知识条目进行排序
|
||||
candidates.sort((a, b) -> { |
||||
// 将字节数组转换回浮点数组
|
||||
float[] embeddingA = EmbeddingService.byteArrayToFloatArray(a.getEmbedding()); |
||||
float[] embeddingB = EmbeddingService.byteArrayToFloatArray(b.getEmbedding()); |
||||
|
||||
// 检查向量是否有效
|
||||
if (embeddingA.length == 0 || embeddingB.length == 0) { |
||||
// 如果任一向量无效，则将该条目排在后面
|
||||
if (embeddingA.length == 0 && embeddingB.length == 0) return 0; |
||||
if (embeddingA.length == 0) return 1; // a排在后面
|
||||
return -1; // b排在后面
|
||||
} |
||||
|
||||
// 检查向量维度是否匹配
|
||||
if (questionEmbedding.length != embeddingA.length || questionEmbedding.length != embeddingB.length) { |
||||
// 如果维度不匹配，则根据维度差异排序，将维度匹配的条目排在前面
|
||||
if (questionEmbedding.length != embeddingA.length && questionEmbedding.length != embeddingB.length) return 0; |
||||
if (questionEmbedding.length != embeddingA.length) return 1; // a排在后面
|
||||
if (questionEmbedding.length != embeddingB.length) return -1; // b排在后面
|
||||
return 0; |
||||
} |
||||
|
||||
// 计算与问题向量的欧几里得距离
|
||||
double distanceA = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embeddingA); |
||||
double distanceB = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embeddingB); |
||||
|
||||
// 按距离升序排序（距离越小越相似）
|
||||
return Double.compare(distanceA, distanceB); |
||||
}); |
||||
|
||||
// 过滤掉没有有效向量或相似度不够高的条目
|
||||
// 设置一个合理的相似度阈值，欧几里得距离越小越相似
|
||||
final double SIMILARITY_THRESHOLD = 100.0; |
||||
|
||||
List<KnowledgeEntity> validEntries = new ArrayList<>(); |
||||
for (KnowledgeEntity entry : candidates) { |
||||
float[] embedding = EmbeddingService.byteArrayToFloatArray(entry.getEmbedding()); |
||||
// 检查向量是否有效且维度匹配
|
||||
if (embedding.length == 0 || embedding.length != questionEmbedding.length) { |
||||
continue; |
||||
} |
||||
|
||||
// 计算与问题的欧几里得距离
|
||||
double distance = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embedding); |
||||
// 只返回距离小于阈值的条目
|
||||
if (distance < SIMILARITY_THRESHOLD) { |
||||
validEntries.add(entry); |
||||
} |
||||
} |
||||
|
||||
// 返回最相似的前limit个条目
|
||||
List<KnowledgeEntity> result = new ArrayList<>(); |
||||
int count = 0; |
||||
for (KnowledgeEntity entry : validEntries) { |
||||
if (count >= limit) { |
||||
break; |
||||
} |
||||
result.add(entry); |
||||
count++; |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* 查找与问题最相似的知识条目 |
||||
* @param question 用户问题 |
||||
* @param limit 返回结果数量上限 |
||||
* @return 相似知识条目列表 |
||||
*/ |
||||
public List<KnowledgeEntity> findSimilarKnowledge(String question, int limit) { |
||||
// 使用关键词和向量混合检索
|
||||
return findSimilarKnowledgeWithKeywordFilter(question, limit); |
||||
} |
||||
|
||||
/** |
||||
* 生成上下文内容 |
||||
* @param knowledgeEntries 知识条目列表 |
||||
* @return 上下文字符串 |
||||
*/ |
||||
public String generateContext(List<KnowledgeEntity> knowledgeEntries) { |
||||
// 检查输入参数
|
||||
if (knowledgeEntries == null || knowledgeEntries.isEmpty()) { |
||||
System.out.println("知识条目列表为空，无法生成上下文"); |
||||
return ""; |
||||
} |
||||
|
||||
StringBuilder context = new StringBuilder(); |
||||
|
||||
for (int i = 0; i < knowledgeEntries.size(); i++) { |
||||
KnowledgeEntity entry = knowledgeEntries.get(i); |
||||
// 检查每个条目及其内容是否为空
|
||||
if (entry != null && entry.getContent() != null && !entry.getContent().trim().isEmpty()) { |
||||
context.append(entry.getContent()); |
||||
|
||||
if (i < knowledgeEntries.size() - 1) { |
||||
context.append("\n\n"); |
||||
} |
||||
} |
||||
} |
||||
|
||||
String result = context.toString().trim(); |
||||
System.out.println("生成上下文，长度: " + result.length()); |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* 更新指定ID的知识条目 |
||||
* @param id 知识条目ID |
||||
* @param content 知识内容 |
||||
* @param source 知识来源 |
||||
* @return 更新是否成功 |
||||
*/ |
||||
@Transactional |
||||
public boolean updateKnowledge(Long id, String content, String source) { |
||||
// 检查知识条目是否存在
|
||||
if (knowledgeRepository.existsById(id)) { |
||||
// 获取现有知识条目
|
||||
KnowledgeEntity existingEntry = knowledgeRepository.findById(id).orElse(null); |
||||
if (existingEntry != null) { |
||||
// 更新内容和来源
|
||||
existingEntry.setContent(content); |
||||
existingEntry.setSource(source); |
||||
|
||||
// 重新计算向量嵌入
|
||||
float[] embedding = embeddingService.getEmbedding(content); |
||||
if (embedding.length > 0) { |
||||
byte[] embeddingBytes = EmbeddingService.floatArrayToByteArray(embedding); |
||||
existingEntry.setEmbedding(embeddingBytes); |
||||
} |
||||
|
||||
// 保存更新后的条目
|
||||
knowledgeRepository.save(existingEntry); |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/** |
||||
* 处理用户提问 |
||||
* @param question 用户问题 |
||||
* @return 答案响应 |
||||
*/ |
||||
public AnswerResponse processQuestion(String question) { |
||||
// 这里应该实现具体的问答逻辑
|
||||
// 由于这是示例，我们返回一个模拟的答案
|
||||
return new AnswerResponse("这是模拟的答案", new String[]{"来源1", "来源2"}); |
||||
} |
||||
|
||||
/** |
||||
* 处理文件上传 |
||||
* @param file 上传的文件 |
||||
* @return 处理结果 |
||||
*/ |
||||
public String handleFileUpload(MultipartFile file) { |
||||
// 这里应该实现具体的文件处理逻辑
|
||||
return "文件上传成功"; |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,298 @@ |
||||
package com.alops.chat.service; |
||||
|
||||
import java.util.*; |
||||
import java.util.stream.Collectors; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
import com.fasterxml.jackson.core.type.TypeReference; |
||||
import com.fasterxml.jackson.databind.JsonNode; |
||||
import org.springframework.http.HttpEntity; |
||||
import org.springframework.http.HttpHeaders; |
||||
import org.springframework.http.MediaType; |
||||
import org.springframework.http.ResponseEntity; |
||||
import org.springframework.web.client.RestTemplate; |
||||
|
||||
@Service |
||||
public class TextSplitter { |
||||
|
||||
private static final int MIN_CHUNK_SIZE = 100; // 最小字数
|
||||
private static final int MAX_CHUNK_SIZE = 500; // 最大字数
|
||||
private static final double SIMILARITY_THRESHOLD = 0.6; // 相似度阈值
|
||||
// Ollama nomic-embed-text:v1.5模型的维度
|
||||
private static final int EMBEDDING_DIMENSION = 768; // nomic-embed-text:v1.5模型的维度
|
||||
|
||||
@Autowired |
||||
private RestTemplate restTemplate; |
||||
|
||||
private static final String OLLAMA_EMBEDDING_URL = "http://localhost:11434/api/embeddings"; |
||||
private static final String EMBEDDING_MODEL = "nomic-embed-text:v1.5"; |
||||
|
||||
private final ObjectMapper objectMapper; |
||||
|
||||
public TextSplitter() { |
||||
this.objectMapper = new ObjectMapper(); |
||||
} |
||||
|
||||
/** |
||||
* 主流程：输入文本，返回分割后的 chunks |
||||
*/ |
||||
public List<String> splitText(String text) { |
||||
// 1. 去除特殊字符
|
||||
String cleanedText = removeSpecialCharacters(text); |
||||
|
||||
// 2. 预切分
|
||||
List<String> units = preSplit(cleanedText); |
||||
|
||||
// 3. 调用 embedding 模型（使用Ollama的nomic-embed-text:v1.5模型）
|
||||
List<float[]> embeddings = units.stream() |
||||
.map(this::getEmbeddingFromOllama) |
||||
.collect(Collectors.toList()); |
||||
|
||||
// 4. 检测候选边界
|
||||
List<Integer> boundaries = detectBoundaries(embeddings, SIMILARITY_THRESHOLD); |
||||
|
||||
// 5. 精修边界（BayesSeg 简化）
|
||||
boundaries = refineBoundaries(boundaries, units); |
||||
|
||||
// 6. 根据边界切分
|
||||
List<String> chunks = cutByBoundaries(units, boundaries); |
||||
|
||||
// 7. 长度控制（合并/再切）
|
||||
return adjustChunks(chunks); |
||||
} |
||||
|
||||
/** |
||||
* 去除特殊字符 |
||||
* @param text 原始文本 |
||||
* @return 清理后的文本 |
||||
*/ |
||||
private String removeSpecialCharacters(String text) { |
||||
if (text == null || text.isEmpty()) { |
||||
return text; |
||||
} |
||||
|
||||
// 定义需要去除的特殊字符（包括但不限于"- ~ ^"）
|
||||
// 可以根据需要添加更多特殊字符到这个列表中
|
||||
String specialChars = "-~^`@#$&*()_=|\\{}'<>./"; |
||||
|
||||
// 创建正则表达式模式来匹配这些特殊字符
|
||||
String regex = "[" + java.util.regex.Pattern.quote(specialChars) + "]+"; |
||||
|
||||
// 将连续的特殊字符替换为单个空格，并去除首尾空格
|
||||
return text.replaceAll(regex, " ").trim(); |
||||
} |
||||
|
||||
/** |
||||
* 预切分：按段落、句子切 |
||||
*/ |
||||
private List<String> preSplit(String text) { |
||||
String[] paras = text.split("\\n\\s*\\n"); // 按空行切段
|
||||
List<String> units = new ArrayList<>(); |
||||
for (String p : paras) { |
||||
if (p.length() > 1000) { |
||||
// 长段再按句子切
|
||||
units.addAll(Arrays.asList(p.split("(?<=[。！？])"))); |
||||
} else { |
||||
units.add(p.trim()); |
||||
} |
||||
} |
||||
return units.stream().filter(s -> !s.isEmpty()).collect(Collectors.toList()); |
||||
} |
||||
|
||||
/** |
||||
* 相似度检测（TopicSeg 思路） |
||||
*/ |
||||
private List<Integer> detectBoundaries(List<float[]> embeddings, double threshold) { |
||||
List<Integer> boundaries = new ArrayList<>(); |
||||
for (int i = 0; i < embeddings.size() - 1; i++) { |
||||
double sim = cosine(embeddings.get(i), embeddings.get(i + 1)); |
||||
if (sim < threshold) { |
||||
boundaries.add(i + 1); |
||||
} |
||||
} |
||||
return boundaries; |
||||
} |
||||
|
||||
/** |
||||
* 精修候选边界（简化版 BayesSeg） |
||||
*/ |
||||
private List<Integer> refineBoundaries(List<Integer> boundaries, List<String> units) { |
||||
List<Integer> refined = new ArrayList<>(); |
||||
for (int b : boundaries) { |
||||
if (b <= 0 || b >= units.size()) continue; |
||||
String left = units.get(b - 1); |
||||
String right = units.get(b); |
||||
if (keepBoundary(left, right)) { |
||||
refined.add(b); |
||||
} |
||||
} |
||||
return refined; |
||||
} |
||||
|
||||
/** |
||||
* 关键词差异判断是否保留边界 |
||||
*/ |
||||
private boolean keepBoundary(String left, String right) { |
||||
Set<String> leftWords = new HashSet<>(Arrays.asList(left.split("\\s+"))); |
||||
Set<String> rightWords = new HashSet<>(Arrays.asList(right.split("\\s+"))); |
||||
long common = leftWords.stream().filter(rightWords::contains).count(); |
||||
double ratio = (double) common / Math.max(1, leftWords.size()); |
||||
return ratio < 0.3; // 共同词过少 → 保留边界
|
||||
} |
||||
|
||||
/** |
||||
* 根据边界切分 |
||||
*/ |
||||
private List<String> cutByBoundaries(List<String> units, List<Integer> boundaries) { |
||||
List<String> chunks = new ArrayList<>(); |
||||
StringBuilder buffer = new StringBuilder(); |
||||
int boundaryIdx = 0; |
||||
for (int i = 0; i < units.size(); i++) { |
||||
buffer.append(units.get(i)).append(" "); |
||||
if (boundaryIdx < boundaries.size() && i + 1 == boundaries.get(boundaryIdx)) { |
||||
chunks.add(buffer.toString().trim()); |
||||
buffer.setLength(0); |
||||
boundaryIdx++; |
||||
} |
||||
} |
||||
if (buffer.length() > 0) chunks.add(buffer.toString().trim()); |
||||
return chunks; |
||||
} |
||||
|
||||
/** |
||||
* 长度控制 |
||||
*/ |
||||
private List<String> adjustChunks(List<String> chunks) { |
||||
List<String> result = new ArrayList<>(); |
||||
StringBuilder buffer = new StringBuilder(); |
||||
|
||||
for (String chunk : chunks) { |
||||
if (chunk.length() < MIN_CHUNK_SIZE && !result.isEmpty()) { |
||||
// 太短 → 合并到前一个
|
||||
int lastIdx = result.size() - 1; |
||||
result.set(lastIdx, result.get(lastIdx) + " " + chunk); |
||||
} else if (chunk.length() > MAX_CHUNK_SIZE) { |
||||
// 太长 → 按句子边界再切分
|
||||
List<String> subChunks = splitLongChunk(chunk); |
||||
result.addAll(subChunks); |
||||
} else { |
||||
result.add(chunk); |
||||
} |
||||
} |
||||
|
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* 分割长文本块，尽量在句子边界处分割 |
||||
* @param chunk 需要分割的长文本 |
||||
* @return 分割后的文本块列表 |
||||
*/ |
||||
private List<String> splitLongChunk(String chunk) { |
||||
List<String> subChunks = new ArrayList<>(); |
||||
|
||||
// 定义句子结束标点符号
|
||||
String sentenceEndings = "。！？.!?"; |
||||
|
||||
// 如果文本长度不超过最大块大小，直接返回
|
||||
if (chunk.length() <= MAX_CHUNK_SIZE) { |
||||
subChunks.add(chunk); |
||||
return subChunks; |
||||
} |
||||
|
||||
int start = 0; |
||||
while (start < chunk.length()) { |
||||
// 计算当前分割点
|
||||
int end = Math.min(start + MAX_CHUNK_SIZE, chunk.length()); |
||||
|
||||
// 如果已经到达文本末尾，直接添加剩余部分
|
||||
if (end >= chunk.length()) { |
||||
subChunks.add(chunk.substring(start)); |
||||
break; |
||||
} |
||||
|
||||
// 寻找最近的句子结束位置
|
||||
int bestSplitPoint = end; |
||||
for (int i = end - 1; i > start; i--) { |
||||
if (sentenceEndings.indexOf(chunk.charAt(i)) != -1) { |
||||
bestSplitPoint = i + 1; // 包含句子结束符号
|
||||
break; |
||||
} |
||||
} |
||||
|
||||
// 如果找到的分割点太靠前（小于最小块大小），则使用固定长度分割
|
||||
if (bestSplitPoint - start < MIN_CHUNK_SIZE) { |
||||
bestSplitPoint = end; |
||||
} |
||||
|
||||
// 添加分割的文本块
|
||||
subChunks.add(chunk.substring(start, bestSplitPoint).trim()); |
||||
start = bestSplitPoint; |
||||
|
||||
// 跳过前面的空白字符
|
||||
while (start < chunk.length() && Character.isWhitespace(chunk.charAt(start))) { |
||||
start++; |
||||
} |
||||
} |
||||
|
||||
return subChunks; |
||||
} |
||||
|
||||
/** |
||||
* 从Ollama获取嵌入向量 |
||||
* 使用nomic-embed-text:v1.5模型 |
||||
*/ |
||||
private float[] getEmbeddingFromOllama(String text) { |
||||
try { |
||||
// 构造请求体
|
||||
Map<String, String> requestBody = new HashMap<>(); |
||||
requestBody.put("model", EMBEDDING_MODEL); |
||||
requestBody.put("prompt", text); |
||||
|
||||
// 设置请求头
|
||||
HttpHeaders headers = new HttpHeaders(); |
||||
headers.setContentType(MediaType.APPLICATION_JSON); |
||||
|
||||
// 创建HTTP实体
|
||||
HttpEntity<Map<String, String>> requestEntity = new HttpEntity<>(requestBody, headers); |
||||
|
||||
// 发送POST请求到Ollama API
|
||||
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_EMBEDDING_URL, requestEntity, String.class); |
||||
|
||||
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { |
||||
// 解析响应
|
||||
JsonNode rootNode = objectMapper.readTree(response.getBody()); |
||||
JsonNode embeddingNode = rootNode.get("embedding"); |
||||
|
||||
if (embeddingNode != null && embeddingNode.isArray()) { |
||||
float[] embeddingArray = new float[embeddingNode.size()]; |
||||
for (int i = 0; i < embeddingNode.size(); i++) { |
||||
embeddingArray[i] = embeddingNode.get(i).floatValue(); |
||||
} |
||||
return embeddingArray; |
||||
} |
||||
} |
||||
|
||||
// 如果无法获取嵌入向量，返回零向量
|
||||
return new float[EMBEDDING_DIMENSION]; // nomic-embed-text:v1.5默认维度为768
|
||||
} catch (Exception e) { |
||||
System.err.println("调用Ollama API获取嵌入向量时出错: " + e.getMessage()); |
||||
// 出错时返回零向量
|
||||
return new float[EMBEDDING_DIMENSION]; // nomic-embed-text:v1.5默认维度为768
|
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 计算余弦相似度 |
||||
*/ |
||||
private double cosine(float[] v1, float[] v2) { |
||||
double dot = 0, normA = 0, normB = 0; |
||||
for (int i = 0; i < v1.length; i++) { |
||||
dot += v1[i] * v2[i]; |
||||
normA += v1[i] * v1[i]; |
||||
normB += v2[i] * v2[i]; |
||||
} |
||||
return dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-10); |
||||
} |
||||
} |
||||
@ -0,0 +1,52 @@ |
||||
package com.alops.chat.service; |
||||
|
||||
import com.alops.chat.entity.UploadedFile; |
||||
import com.alops.chat.repository.UploadedFileRepository; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* 上传文件信息服务类 |
||||
* 提供对上传文件信息的业务处理 |
||||
*/ |
||||
@Service |
||||
public class UploadedFileService { |
||||
|
||||
@Autowired |
||||
private UploadedFileRepository uploadedFileRepository; |
||||
|
||||
/** |
||||
* 保存上传文件信息 |
||||
* @param uploadedFile 上传文件信息实体 |
||||
* @return 保存后的实体 |
||||
*/ |
||||
public UploadedFile save(UploadedFile uploadedFile) { |
||||
return uploadedFileRepository.save(uploadedFile); |
||||
} |
||||
|
||||
/** |
||||
* 根据ID查找上传文件信息 |
||||
* @param id 文件ID |
||||
* @return 上传文件信息实体 |
||||
*/ |
||||
public UploadedFile findById(Long id) { |
||||
return uploadedFileRepository.findById(id).orElse(null); |
||||
} |
||||
|
||||
/** |
||||
* 获取所有上传文件信息 |
||||
* @return 上传文件信息列表 |
||||
*/ |
||||
public List<UploadedFile> findAll() { |
||||
return uploadedFileRepository.findAll(); |
||||
} |
||||
|
||||
/** |
||||
* 删除上传文件信息 |
||||
* @param id 文件ID |
||||
*/ |
||||
public void deleteById(Long id) { |
||||
uploadedFileRepository.deleteById(id); |
||||
} |
||||
} |
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in new issue