/*
 *  Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com).
 *  <p>
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *  <p>
 *  http://www.apache.org/licenses/LICENSE-2.0
 *  <p>
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.easyagents.core.document.splitter;

import com.easyagents.core.document.Document;
import com.easyagents.core.document.DocumentSplitter;
import com.easyagents.core.document.id.DocumentIdGenerator;
import com.easyagents.core.util.StringUtil;
import com.knuddels.jtokkit.Encodings;
import com.knuddels.jtokkit.api.Encoding;
import com.knuddels.jtokkit.api.EncodingRegistry;
import com.knuddels.jtokkit.api.EncodingType;
import com.knuddels.jtokkit.api.IntArrayList;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

public class SimpleTokenizeSplitter implements DocumentSplitter {
    private EncodingRegistry registry = Encodings.newLazyEncodingRegistry();
    private EncodingType encodingType = EncodingType.CL100K_BASE;
    private int chunkSize;
    private int overlapSize;

    public SimpleTokenizeSplitter(int chunkSize) {
        this.chunkSize = chunkSize;
        if (this.chunkSize <= 0) {
            throw new IllegalArgumentException("chunkSize must be greater than 0, chunkSize: " + this.chunkSize);
        }
    }

    public SimpleTokenizeSplitter(int chunkSize, int overlapSize) {
        this.chunkSize = chunkSize;
        this.overlapSize = overlapSize;

        if (this.chunkSize <= 0) {
            throw new IllegalArgumentException("chunkSize must be greater than 0, chunkSize: " + this.chunkSize);
        }
        if (this.overlapSize >= this.chunkSize) {
            throw new IllegalArgumentException("overlapSize must be less than chunkSize, overlapSize: " + this.overlapSize + ", chunkSize: " + this.chunkSize);
        }
    }

    public int getChunkSize() {
        return chunkSize;
    }

    public void setChunkSize(int chunkSize) {
        this.chunkSize = chunkSize;
    }

    public int getOverlapSize() {
        return overlapSize;
    }

    public void setOverlapSize(int overlapSize) {
        this.overlapSize = overlapSize;
    }

    public EncodingRegistry getRegistry() {
        return registry;
    }

    public void setRegistry(EncodingRegistry registry) {
        this.registry = registry;
    }

    public EncodingType getEncodingType() {
        return encodingType;
    }

    public void setEncodingType(EncodingType encodingType) {
        this.encodingType = encodingType;
    }

    @Override
    public List<Document> split(Document document, DocumentIdGenerator idGenerator) {
        if (document == null || StringUtil.noText(document.getContent())) {
            return Collections.emptyList();
        }

        String content = document.getContent();
        Encoding encoding = this.registry.getEncoding(this.encodingType);

        List<Integer> tokens = encoding.encode(content).boxed();


        int index = 0, currentIndex = index;
        int maxIndex = tokens.size();

        List<Document> chunks = new ArrayList<>();
        while (currentIndex < maxIndex) {
            int endIndex = Math.min(currentIndex + chunkSize, maxIndex);
            List<Integer> chunkTokens = tokens.subList(currentIndex, endIndex);

            IntArrayList intArrayList = new IntArrayList();
            for (Integer chunkToken : chunkTokens) {
                intArrayList.add(chunkToken);
            }
            String chunkText = encoding.decode(intArrayList).trim();
            if (chunkText.isEmpty()) {
                continue;
            }

            //UTF-8 'Unicode replacement character' which in your case is 0xFFFD (65533 in Hex).
            //fix 修复中文乱码的问题
            boolean firstIsReplacement = chunkText.charAt(0) == 65533;
            boolean lastIsReplacement = chunkText.charAt(chunkText.length() - 1) == 65533;

            if (firstIsReplacement || lastIsReplacement) {
                if (firstIsReplacement) currentIndex -= 1;
                if (lastIsReplacement) endIndex += 1;

                chunkTokens = tokens.subList(currentIndex, endIndex);
                intArrayList = new IntArrayList();
                for (Integer chunkToken : chunkTokens) {
                    intArrayList.add(chunkToken);
                }

                chunkText = encoding.decode(intArrayList).trim();
            }

            currentIndex = currentIndex + chunkSize - overlapSize;

            Document newDocument = new Document();
            newDocument.addMetadata(document.getMetadataMap());
            newDocument.setContent(chunkText);

            //we should invoke setId after setContent
            newDocument.setId(idGenerator == null ? null : idGenerator.generateId(newDocument));
            chunks.add(newDocument);
        }

        return chunks;
    }
}
