[DERCBOT-1609] Improving RAG System - Part 1

bot/admin/server/src/main/kotlin/model/genai/BotRAGConfigurationDTO.kt

@@ -42,9 +42,6 @@ data class BotRAGConfigurationDTO(
     val emSetting: EMSettingDTO,
     val indexSessionId: String? = null,
     val indexName: String? = null,
-    val noAnswerSentence: String,
-    val noAnswerStoryId: String? = null,
-    val documentsRequired: Boolean = true,
     val debugEnabled: Boolean,
     val maxDocumentsRetrieved: Int,
     val maxMessagesFromHistory: Int,
@@ -63,9 +60,6 @@ data class BotRAGConfigurationDTO(
         emSetting = configuration.emSetting.toDTO(),
         indexSessionId = configuration.indexSessionId,
         indexName = configuration.generateIndexName(),
-        noAnswerSentence = configuration.noAnswerSentence,
-        noAnswerStoryId = configuration.noAnswerStoryId,
-        documentsRequired = configuration.documentsRequired,
         debugEnabled = configuration.debugEnabled,
         maxDocumentsRetrieved = configuration.maxDocumentsRetrieved,
         maxMessagesFromHistory = configuration.maxMessagesFromHistory,
@@ -101,9 +95,6 @@ data class BotRAGConfigurationDTO(
                     dto = emSetting,
                 ),
             indexSessionId = indexSessionId,
-            noAnswerSentence = noAnswerSentence,
-            noAnswerStoryId = noAnswerStoryId,
-            documentsRequired = documentsRequired,
             debugEnabled = debugEnabled,
             maxDocumentsRetrieved = maxDocumentsRetrieved,
             maxMessagesFromHistory = maxMessagesFromHistory,

bot/admin/server/src/test/kotlin/service/RAGServiceTest.kt

@@ -97,8 +97,6 @@ class RAGServiceTest : AbstractTest() {
                         model = "model",
                         apiBase = "url",
                     ),
-                noAnswerSentence = "No answer sentence",
-                documentsRequired = true,
                 debugEnabled = false,
                 maxDocumentsRetrieved = 2,
                 maxMessagesFromHistory = 2,
@@ -211,7 +209,6 @@ class RAGServiceTest : AbstractTest() {
             Assertions.assertEquals(PROVIDER, captured.questionAnsweringLlmSetting!!.provider.name)
             Assertions.assertEquals(TEMPERATURE, captured.questionAnsweringLlmSetting!!.temperature)
             Assertions.assertEquals(PROMPT, captured.questionAnsweringPrompt!!.template)
-            Assertions.assertEquals(null, captured.noAnswerStoryId)
         }
 
         TestCase<SaveFnEntry, Unit>("Save valid RAG Configuration that does not exist yet").given(

bot/admin/server/src/test/kotlin/service/RAGValidationServiceTest.kt

@@ -90,8 +90,6 @@ class RAGValidationServiceTest {
             questionAnsweringLlmSetting = openAILLMSetting,
             questionAnsweringPrompt = PromptTemplate(template = "How to bike in the rain"),
             emSetting = azureOpenAIEMSetting,
-            noAnswerSentence = " No answer sentence",
-            documentsRequired = true,
             debugEnabled = false,
             maxDocumentsRetrieved = 2,
             maxMessagesFromHistory = 2,

bot/admin/web/src/app/analytics/dialogs/dialogs-list/dialogs-list-filters/dialogs-list-filters.component.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
+import { Component, EventEmitter, Input, OnDestroy, OnInit, Output } from '@angular/core';
 import { FormControl, FormGroup } from '@angular/forms';
 import { ConnectorType } from '../../../../core/model/configuration';
 import { Subject, debounceTime, take, takeUntil } from 'rxjs';
@@ -67,7 +67,7 @@ export type DialogListFilters = ExtractFormControlTyping<DialogListFiltersForm>;
   templateUrl: './dialogs-list-filters.component.html',
   styleUrl: './dialogs-list-filters.component.scss'
 })
-export class DialogsListFiltersComponent implements OnInit {
+export class DialogsListFiltersComponent implements OnInit, OnDestroy {
   private readonly destroy$: Subject<boolean> = new Subject();
   private lastEmittedValue: Partial<DialogListFilters> | null = null;
 
@@ -109,8 +109,9 @@ export class DialogsListFiltersComponent implements OnInit {
       this.lastEmittedValue = { ...this.form.value };
     }
 
-    this.form.valueChanges.pipe(debounceTime(800), takeUntil(this.destroy$)).subscribe(() => {
+    this.form.valueChanges.pipe(debounceTime(500), takeUntil(this.destroy$)).subscribe(() => {
       this.submitFiltersChange();
+      this.persisteDisplayTests();
     });
   }
 
@@ -143,10 +144,15 @@ export class DialogsListFiltersComponent implements OnInit {
 
   submitFiltersChange(): void {
     const formValue = this.form.value;
-    if (JSON.stringify(formValue) !== JSON.stringify(this.lastEmittedValue)) {
-      this.onFilter.emit(formValue);
-      this.lastEmittedValue = { ...formValue };
-    }
+    this.onFilter.emit(formValue);
+  }
+
+  persisteDisplayTests(): void {
+    const displayTests = this.getFormControl('displayTests')?.value;
+    this.botSharedService.session_storage = {
+      ...this.botSharedService.session_storage,
+      ...{ dialogs: { ...this.botSharedService.session_storage?.dialogs, displayTests } }
+    };
   }
 
   resetControl(ctrl: FormControl, input?: HTMLInputElement): void {

bot/admin/web/src/app/analytics/dialogs/dialogs-list/dialogs-list.component.ts

@@ -73,6 +73,10 @@ export class DialogsListComponent implements OnInit, OnChanges, OnDestroy {
   }
 
   ngOnInit() {
+    if (this.botSharedService.session_storage?.dialogs?.displayTests) {
+      this.filters.displayTests = this.botSharedService.session_storage.dialogs.displayTests;
+    }
+
     this.state.configurationChange.pipe(takeUntil(this.destroy$)).subscribe(() => {
       this.refresh();
     });

bot/admin/web/src/app/rag/rag-settings/models/engines-configurations.ts

@@ -27,35 +27,209 @@ import {
   PromptDefinitionFormatter
 } from '../../../shared/model/ai-settings';
 
-export const QuestionCondensingDefaultPrompt: string = `Given a chat history and the latest user question which might reference context in the chat history, formulate a standalone question which can be understood without the chat history. Do NOT answer the question, just reformulate it if needed and otherwise return it as is.`;
+export const QuestionCondensingDefaultPrompt: string = `You are a helpful assistant that reformulates questions.
+
+You are given:
+- The conversation history between the user and the assistant
+- The most recent user question
+
+Your task:
+- Reformulate the user’s latest question into a clear, standalone query.
+- Incorporate relevant context from the conversation history.
+- Do NOT answer the question.
+- If the history does not provide additional context, keep the question as is.
+
+Return only the reformulated question.`;
 
 export const QuestionAnsweringDefaultPrompt: string = `# TOCK (The Open Conversation Kit) chatbot
 
-## General context
+## General Context
 
 You are a chatbot designed to provide short conversational messages in response to user queries.
+Your job is to surface the right information from provided context.
+
+### Forbidden Topics
+
+- **Out of Scope**:
+  - Topics unrelated to the business domain (e.g., personal life advice, unrelated industries).
+  - Requests for unsupported features (e.g., "How do I integrate with [UnsupportedTool]?").
+
+- **Toxic/Offensive Content**:
+  - Hate speech, harassment, or discriminatory language.
+  - Illegal activities or unethical requests (e.g., "How do I bypass security protocols?").
+
+- **Personal/Private Matters**:
+  - User-specific data (e.g., personal identification, private conversations).
+  - Internal or confidential company information (e.g., unreleased product details).
+
+- **Regulated Topics**:
+  - Medical, legal, or financial advice (e.g., "What’s the best treatment for [condition]?").
+  - Speculative or unverified claims (e.g., "Is [Product X] better than competitors?").
+
+### Answer Style
+
+- **Tone**: neutral, kind, “you” address, light humor when appropriate.
+- **Language**: Introduce technical jargon only when strictly necessary and briefly define it.
+- **Structure**: Use short sentences, bold or bullet points for key ideas, headings to separate the main sections, and fenced \`code\` blocks for examples. Only include absolute links in your answers.
+- **Style**: Direct and technical tone, with **bold** for important concepts.
+- **Formatting**: Mandatory Markdown, with line breaks for readability.
+- **Examples**: Include a concrete example (code block or CLI command) for each feature.
+
+### Guidelines
+
+1. If the question is unclear, politely request rephrasing.
+2. If the docs lack or don’t cover the answer, reply with \`"status": "not_found_in_context"\`.
+3. Conclude with:
+   - “Does this help?”
+   - Offer to continue on the same topic, switch topics, or contact support.
+
+### Verification Steps
+
+Before responding, ensure:
+
+- The documentation actually addresses the question.
+- Your answer is consistent with the docs.
+- If you include a link in your response, make sure it is an absolute link; otherwise, do not include it.
+
+## Technical Instructions:
+
+You must respond STRICTLY in valid JSON format (no extra text, no explanations).
+Use only the following context and the rules below to respond the question.
+
+### Rules for JSON output:
+
+- If the answer is found in the context:
+  - "status": "found_in_context"
+  - "answer": the best possible answer in {{ locale }}
+  - "display_answer": "true"
+  - "redirection_intent": null
 
-## Guidelines
+- If the answer is NOT found in the context:
+  - "status": "not_found_in_context"
+  - "answer":
+    - The "answer" must not be a generic refusal. Instead, generate a helpful and intelligent response:
+      - If a similar or related element exists in the context (e.g., another product, service, or regulation with a close name, date, or wording), suggest it naturally in the answer.
+      - If no similar element exists, politely acknowledge the lack of information while encouraging clarification or rephrasing.
+    - Always ensure the response is phrased in a natural and user-friendly way, rather than a dry "not found in context".
+  - "display_answer": "true"
+  - "redirection_intent": null
 
-Incorporate any relevant details from the provided context into your answers, ensuring they are directly related to the user's query.
+- If the question is forbidden or offensive:
+  - "status": "out_of_scope"
+  - "answer":
+    - Generate a polite response explaining why a reponse can't be done.
+  - "topic": "Out of scope or offensive question"
+  - "display_answer": "true"
+  - "redirection_intent": null
 
-## Style and format
+- If the question is small talk:
+  Only to conversational rituals such as greetings (e.g., “hello”, “hi”) and farewells or leave-takings (e.g., “goodbye”, “see you”), you may ignore the context and generate a natural small-talk response in the "answer".
+  - "status": "small_talk"
+  - "topic": "greetings"
+  - "display_answer": "true"
+  - "redirection_intent": null
 
-Your tone is empathetic, informative and polite.
+### Confidence score:
 
-## Additional instructions
+Gives a confidence score between 0 and 1 on the relevance of the answer provided to the user's question:
 
-Use the following pieces of retrieved context to answer the question.
-If you dont know the answer, answer (exactly) with "{{no_answer}}".
-Answer in {{locale}}.
+- "confidence_score": <CONFIDENCE_SCORE>
 
-## Context
+### Users question understanding:
 
-{{context}}
+Explain in one sentence what you understood from the user's question:
 
-## Question
+- "understanding": "<UNDERSTANDING_OF_THE_USER_QUESTION>"
+
+### Context usage tracing requirements (MANDATORY):
+
+- You MUST include **every** chunk from the input context in the "context_usage" array, in the same order they appear. **No chunk may be omitted**.
+- If explicit chunk identifiers are present in the context, use them.
+- For each chunk object:
+  - "chunk": "<chunk_identifier>"
+  - "used_in_response":
+    - "true" if the chunk contributed
+    - "false" if the chunk didn't contributed
+  - "sentences": ["<verbatim sentence(s) from this chunk used to answer the question>"] — leave empty \`[]\` if none.
+  - "reason": "null" if the chunk contributed; otherwise a concise explanation of why this chunk is not relevant to the question (e.g., "general background only", "different product", "no data for the asked period", etc.).
+- If there are zero chunks in the context, return \`"context": []\`.
+
+### Topic Identification & Suggestion Rules (MANDATORY):
+
+#### Rules for Topic Assignment
+
+- If the question explicitly matches a predefined topic, use:
+  - \`"topic": "<EXACT_PREDEFINED_TOPIC>"\`
+  - \`"suggested_topics": []\`
+
+- If the question does not match any predefined topic, use the \`unknown\` topic and provide 1 relevant and concise new topic suggestion in "suggested_topics":
+  - \`"topic": "unknown"\`
+  - \`"suggested_topics": ["<NEW_TOPIC_SUGGESTION>"]\`
+
+#### Predefined topics (use EXACT spelling, no variations):
+
+- \`Concepts and Definitions\`
+- \`Processes and Methods\`
+- \`Tools and Technologies\`
+- \`Rules and Regulations\`
+- \`Examples and Use Cases\`
+- \`Resources and References\`
+
+## Context:
+
+{{ context }}
+
+## User question
+
+You are given the conversation history between a user and an assistant:
+
+- analyze the conversation history to understand the context and the user’s intent
+- use this context to correctly interpret the user’s final question
+- answer only the final user question below in a relevant and contextualized way
+
+Conversation history:
+{{ chat_history }}
+
+User’s final question:
+{{ question }}
+
+## Output format (JSON only):
+
+Return your response in the following format:
+
+\`\`\`json
+{
+    "status": "found_in_context" | "not_found_in_context" | "small_talk" | "out_of_scope",
+    "answer": "<TEXTUAL_ANSWER>",
+    "display_answer": true | false,
+    "confidence_score": "<CONFIDENCE_SCORE>",
+    "topic": "<EXACT_PREDEFINED_TOPIC>" | "greetings" | "Out of scope or offensive question" | "unknown",
+    "suggested_topics": ["<topic_suggestion_1>", "<topic_suggestion_2>"],
+    "understanding": "<UNDERSTANDING_OF_THE_USER_QUESTION>",
+    "redirection_intent": null,
+    "context_usage": [
+        {
+            "chunk": "1",
+            "sentences": ["SENTENCE_1", "SENTENCE_2"],
+            "used_in_response": true | false,
+            "reason": null
+        },
+        {
+            "chunk": "2",
+            "sentences": [],
+            "used_in_response": true | false,
+            "reason": "General description; no details related to the question."
+        },
+        {
+            "chunk": "3",
+            "sentences": ["SENTENCE_X"],
+            "used_in_response": true | false,
+            "reason": null
+        }
+    ]
+}
+\`\`\`
 
-{{question}}
 `;
 
 export const QuestionCondensing_prompt: ProvidersConfigurationParam[] = [

bot/admin/web/src/app/rag/rag-settings/models/rag-settings.ts

@@ -24,9 +24,6 @@ export interface RagSettings {
 
   debugEnabled: boolean;
 
-  noAnswerSentence: string;
-  noAnswerStoryId: string | null;
-
   questionCondensingLlmSetting: llmSetting;
   questionCondensingPrompt: PromptDefinition;
   maxMessagesFromHistory: number;
@@ -40,6 +37,4 @@ export interface RagSettings {
   indexName: string;
 
   maxDocumentsRetrieved: number;
-
-  documentsRequired: boolean;
 }