Chunk sizes

mock_qa_prompt_tmpl = "{context_str}{query_str}"
mock_qa_prompt = QuestionAnswerPrompt(mock_qa_prompt_tmpl)
prompt_helper = PromptHelper(
    max_input_size=11,
    num_output=0,
    max_chunk_overlap=0,
    tokenizer=mock_tokenizer,
    separator="\n\n",
    chunk_size_limit=20,
)
texts = [
    "This",
    "is",
    "bar",
    "This",
    "is",
    "foo",
]
compacted_chunks = prompt_helper.compact_text_chunks(mock_qa_prompt, texts)
assert compacted_chunks == ["This\n\nis\n\nbar", "This\n\nis\n\nfoo"]
# AssertionError: assert ['This', '', 'is', '', 'bar', '', 'This', '', 'is', '', 'foo'] == ['This\n\nis\n\nbar', 'This\n\nis\n\nfoo']

def test_compact_text() -> None:
    """Test compact text."""
    test_prompt_text = "This is the prompt{text}"
    test_prompt = TestPrompt(test_prompt_text)
    prompt_helper = PromptHelper(
        max_input_size=9,
        num_output=1,
        max_chunk_overlap=0,
        tokenizer=mock_tokenizer,
        separator="\n\n",
    )
    text_chunks = ["Hello", "world", "foo", "Hello", "world", "bar"]
    compacted_chunks = prompt_helper.compact_text_chunks(test_prompt, text_chunks)
    assert compacted_chunks == ["Hello\n\nworld\n\nfoo", "Hello\n\nworld\n\nbar"]

>>> from llama_index import PromptHelper
>>> prompt_helper = PromptHelper(max_input_size=4096, num_output=256, max_chunk_overlap=20)
>>> from llama_index.prompts.prompts import QuestionAnswerPrompt
>>> mock_prompt = QuestionAnswerPrompt("{context_str}{query_str}")
>>> prompt_helper.compact_text_chunks(mock_prompt, ['This', 'Should', 'be', 'one', 'msg.'])
['This\n\nShould\n\nbe\n\none\n\nmsg.']
>>> 

def test_accumulate_compact_response(
    mock_service_context: ServiceContext,
) -> None:
  mock_qa_prompt_tmpl = "{context_str}{query_str}"
    mock_qa_prompt = QuestionAnswerPrompt(mock_qa_prompt_tmpl)
    prompt_helper = PromptHelper(
        max_input_size=100,
        num_output=0,
        max_chunk_overlap=0,
        tokenizer=mock_tokenizer,
        separator="\n\n",
    )
    texts = [
        "This",
        "is",
        "bar",
        "This",
        "is",
        "foo",
    ]
    compacted_chunks = prompt_helper.compact_text_chunks(mock_qa_prompt, texts)
    assert compacted_chunks == ["This\n\nis\n\nbar\n\nThis\n\nis\n\nfoo"]

pytest -k test_accumulate_compact_response tests/indices/response/test_response_builder.py -vv

@pytest.fixture()
def mock_service_context(
    patch_token_text_splitter: Any, patch_llm_predictor: Any
) -> ServiceContext:
    return ServiceContext.from_defaults(embed_model=MockEmbedding())

@pytest.fixture
def patch_token_text_splitter(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(TokenTextSplitter, "split_text", patch_token_splitter_newline)
    monkeypatch.setattr(
        TokenTextSplitter,
        "split_text_with_overlaps",
        patch_token_splitter_newline_with_overlaps,
    )