Fix/3748 pasted html tables

src/prerna/reactor/agent/AgentRunner.java

@@ -143,6 +143,10 @@ public final class AgentRunner {
 
 	/** The user's own words from agentParams, else the full input. */
 	public static String userText(Map<String, Object> agentParams, String input) {
+		if (agentParams != null && AgentUserContent.HTML_V1.equals(agentParams.get(AgentUserContent.FORMAT_KEY))
+				&& agentParams.get(AGENT_PARAM_USER_TEXT) instanceof String richText) {
+			return richText;
+		}
 		String userText = agentParams == null ? null : ValueUtils.trimToNull(agentParams.get(AGENT_PARAM_USER_TEXT));
 		return userText != null ? userText : input;
 	}
@@ -226,6 +230,9 @@ private static AgentHarnessResult run(String roomId, String input, String engine
 		if (input == null || input.trim().isEmpty()) {
 			throw new IllegalArgumentException("input is required");
 		}
+		AgentUserContent.Normalized userContent = AgentUserContent.normalize(input, agentParamMap);
+		input = userContent.command();
+		agentParamMap = userContent.params();
 		// One-agent-per-room is enforced by AgentRunRegistry, which every caller goes
 		// through: AgentRunQueueLoop claims the room before dispatching here and
 		// releases

src/prerna/reactor/agent/AgentUserContent.java

@@ -0,0 +1,176 @@
+/*******************************************************************************
+ * Copyright 2015 Defense Health Agency (DHA)
+ *
+ * If your use of this software does not include any GPLv2 components:
+ * 	Licensed under the Apache License, Version 2.0 (the "License");
+ * 	you may not use this file except in compliance with the License.
+ * 	You may obtain a copy of the License at
+ *
+ * 	  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 	Unless required by applicable law or agreed to in writing, software
+ * 	distributed under the License is distributed on an "AS IS" BASIS,
+ * 	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * 	See the License for the specific language governing permissions and
+ * 	limitations under the License.
+ * ----------------------------------------------------------------------------
+ * If your use of this software includes any GPLv2 components:
+ * 	This program is free software; you can redistribute it and/or
+ * 	modify it under the terms of the GNU General Public License
+ * 	as published by the Free Software Foundation; either version 2
+ * 	of the License, or (at your option) any later version.
+ *
+ * 	This program is distributed in the hope that it will be useful,
+ * 	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * 	GNU General Public License for more details.
+ *******************************************************************************/
+package prerna.reactor.agent;
+
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.parser.Parser;
+import org.jsoup.safety.Cleaner;
+import org.jsoup.safety.Safelist;
+
+import prerna.engine.impl.model.message.InputMessage;
+
+/** Explicit, bounded semantic HTML input shared by composer, history and model. */
+public final class AgentUserContent {
+
+	public static final String FORMAT_KEY = "userContentFormat";
+	public static final String CONTENT_KEY = "userContent";
+	public static final String HTML_V1 = "html-v1";
+	public static final int MAX_BYTES = 262144;
+	public static final int MAX_TEXT = 100000;
+	public static final int MAX_CELLS = 5000;
+	public static final int MAX_SPAN = 100;
+	public static final int MAX_DEPTH = 32;
+	public static final int MAX_NODES = 20000;
+	private static final Set<String> SCOPES = Set.of("row", "col", "rowgroup", "colgroup");
+	private AgentUserContent() {
+	}
+
+	public static boolean isRich(Map<String, Object> params) {
+		return params != null && params.containsKey(FORMAT_KEY);
+	}
+
+	/** Pixel has decoded explicit rich commands once; retain legacy decoding otherwise. */
+	public static String decodeCommand(String input, Map<String, Object> params) {
+		if (input == null || isRich(params) || input.indexOf('%') < 0) {
+			return input;
+		}
+		try {
+			return URLDecoder.decode(input, StandardCharsets.UTF_8);
+		} catch (IllegalArgumentException e) {
+			return input;
+		}
+	}
+
+	public record Normalized(String command, Map<String, Object> params) {
+	}
+
+	/** Reject inconsistent rich requests before they reach durable submission or a provider. */
+	public static Normalized normalize(String command, Map<String, Object> params) {
+		if (!isRich(params)) {
+			return new Normalized(command, params);
+		}
+		if (!HTML_V1.equals(params.get(FORMAT_KEY))) {
+			throw new IllegalArgumentException("Unsupported userContentFormat");
+		}
+		if (!(params.get(CONTENT_KEY) instanceof String content) || content.isBlank()) {
+			throw new IllegalArgumentException("Rich userContent is required");
+		}
+		if (!content.equals(params.get(AgentRunner.AGENT_PARAM_USER_TEXT))) {
+			throw new IllegalArgumentException("Rich userText must equal userContent");
+		}
+		if (command == null || !command.endsWith(content)) {
+			throw new IllegalArgumentException("Rich command must end with userContent unchanged");
+		}
+		String clean = sanitize(content);
+		Map<String, Object> normalized = new HashMap<>(params);
+		normalized.put(CONTENT_KEY, clean);
+		normalized.put(AgentRunner.AGENT_PARAM_USER_TEXT, clean);
+		return new Normalized(command.substring(0, command.length() - content.length()) + clean, normalized);
+	}
+
+	public static String sanitize(String content) {
+		if (content.length() > MAX_BYTES || content.getBytes(StandardCharsets.UTF_8).length > MAX_BYTES) {
+			throw new IllegalArgumentException("Rich userContent exceeds the byte limit");
+		}
+		Parser parser = Parser.htmlParser().setTrackErrors(1);
+		Document dirty = parser.parseInput(content, "");
+		if (!parser.getErrors().isEmpty()) {
+			throw new IllegalArgumentException("Rich userContent contains malformed HTML");
+		}
+		// Validate the complete source tree before cleaning; dropped tags cannot evade limits.
+		int[] counts = new int[3];
+		validateTree(dirty.body(), 0, false, counts);
+		if (dirty.body().wholeText().length() > MAX_TEXT) {
+			throw new IllegalArgumentException("Rich userContent exceeds the text limit");
+		}
+		Safelist allowed = Safelist.none().addTags("p", "br", "strong", "em", "u", "s", "blockquote", "ul",
+				"ol", "li", "pre", "code", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td")
+				.addAttributes("td", "colspan", "rowspan").addAttributes("th", "colspan", "rowspan", "scope");
+		Document safe = new Cleaner(allowed).clean(dirty);
+		safe.outputSettings().prettyPrint(false).charset(StandardCharsets.UTF_8);
+		String canonical = safe.body().html();
+		if (safe.body().text().isBlank()) {
+			throw new IllegalArgumentException("Rich userContent must contain text");
+		}
+		if (canonical.getBytes(StandardCharsets.UTF_8).length > MAX_BYTES) {
+			throw new IllegalArgumentException("Canonical rich userContent exceeds the byte limit");
+		}
+		return canonical;
+	}
+
+	private static void validateTree(Node node, int depth, boolean insideTable, int[] counts) {
+		if (depth > MAX_DEPTH || ++counts[0] > MAX_NODES) {
+			throw new IllegalArgumentException("Rich userContent exceeds structural limits");
+		}
+		boolean table = node instanceof Element element && "table".equals(element.normalName());
+		if (table && insideTable) {
+			throw new IllegalArgumentException("Nested rich tables are not supported");
+		}
+		if (node instanceof Element element && Set.of("td", "th").contains(element.normalName())) {
+			if (++counts[1] > MAX_CELLS) {
+				throw new IllegalArgumentException("Rich userContent exceeds the cell limit");
+			}
+			int expandedSlots = 1;
+			for (String attribute : new String[] { "colspan", "rowspan" }) {
+				if (element.hasAttr(attribute)) {
+					String value = element.attr(attribute);
+					if (!value.matches("[0-9]{1,3}") || Integer.parseInt(value) < 1
+							|| Integer.parseInt(value) > MAX_SPAN) {
+						throw new IllegalArgumentException("Rich table span must be between 1 and " + MAX_SPAN);
+					}
+					expandedSlots *= Integer.parseInt(value);
+				}
+			}
+			counts[2] += expandedSlots;
+			if (counts[2] > MAX_CELLS) {
+				throw new IllegalArgumentException("Rich userContent exceeds the expanded table cell limit");
+			}
+			if (element.hasAttr("scope") && !SCOPES.contains(element.attr("scope"))) {
+				throw new IllegalArgumentException("Unsupported rich table header scope");
+			}
+		}
+		for (Node child : node.childNodes()) {
+			validateTree(child, depth + 1, insideTable || table, counts);
+		}
+	}
+
+	/** Only the explicit contract opts the saved user message into rich rendering. */
+	public static void annotate(InputMessage message, Map<String, Object> params) {
+		if (params != null && HTML_V1.equals(params.get(FORMAT_KEY))) {
+			message.setOrnament(FORMAT_KEY, HTML_V1);
+		}
+	}
+}

src/prerna/reactor/agent/RunAgentReactor.java

@@ -27,8 +27,6 @@
  *******************************************************************************/
 package prerna.reactor.agent;
 
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -92,15 +90,6 @@ public NounMetadata execute() {
 		String harnessType = this.keyValue.get(HARNESS_TYPE_KEY);
 		String requestedSpace = StringUtils.trimToNull(this.keyValue.get(SPACE_KEY));
 
-		// FE sends `command` URL-encoded (spaces as %20, etc.). Decode before
-		// forwarding to the harness so the prompt reaches the model intact.
-		if (input != null && input.indexOf('%') >= 0) {
-			try {
-				input = URLDecoder.decode(input, StandardCharsets.UTF_8);
-			} catch (IllegalArgumentException e) {
-				logger.warn("RunAgentReactor: command failed URL-decode, passing through raw: {}", e.getMessage());
-			}
-		}
 		// workspaceId overrides room.options.workspace.workspace_id for this run.
 		String explicitWorkspaceId = StringUtils.trimToNull(this.keyValue.get(WORKSPACE_ID_KEY));
 
@@ -111,6 +100,10 @@ public NounMetadata execute() {
 		long waitTimeoutMs = parseLongAtLeast(this.keyValue.get(WAIT_TIMEOUT_MS_KEY), 0L, 0L);
 		Map<String, Object> paramMap = getMap("paramMap");
 		Map<String, Object> agentParams = getMap("agentParams");
+		input = AgentUserContent.decodeCommand(input, agentParams);
+		AgentUserContent.Normalized userContent = AgentUserContent.normalize(input, agentParams);
+		input = userContent.command();
+		agentParams = userContent.params();
 		if (paramMap.containsKey(AgentRunner.PARAM_SPACE)) {
 			throw new IllegalArgumentException(
 					"RunAgent space must be provided as a top-level argument, not paramValues.space");

src/prerna/reactor/agent/run/AgentRunRequest.java

@@ -36,6 +36,7 @@
 import prerna.om.Insight;
 import prerna.reactor.agent.AgentRunContext;
 import prerna.reactor.agent.AgentRunner;
+import prerna.reactor.agent.AgentUserContent;
 import prerna.util.ValueUtils;
 
 /**
@@ -173,12 +174,25 @@ public AgentRunRequest withCompletionMode(SubAgentRunCompletionMode completionMo
 				ownerAuthType, continuationChildRunId, continuationDepth, humanExecutorLabel);
 	}
 
+	/** Normalize explicit rich input before persisting the durable request. */
+	AgentRunRequest withNormalizedUserContent() {
+		if (!AgentUserContent.isRich(agentParamMap)) {
+			return this;
+		}
+		AgentUserContent.Normalized content = AgentUserContent.normalize(input, agentParamMap);
+		return new AgentRunRequest(roomId, content.command(), engineIdFallback, harnessType, workspaceId, maxTurns,
+				maxReflections, paramMap, content.params(), mediaInputPaths, mediaUrls, insight, resumeMode, parentRunId,
+				completionMode, ownerAuthType, continuationChildRunId, continuationDepth, humanExecutorLabel);
+	}
+
 	/** Builds a fresh same-room run that follows one detached child result. */
 	AgentRunRequest forContinuation(String parentRoomId, String childRunId, String continuationInput,
 			Insight continuationInsight) {
 		// the user's text belonged to the original request, not the continuation
 		Map<String, Object> continuationAgentParams = new HashMap<>(agentParamMap);
 		continuationAgentParams.remove(AgentRunner.AGENT_PARAM_USER_TEXT);
+		continuationAgentParams.remove(AgentUserContent.CONTENT_KEY);
+		continuationAgentParams.remove(AgentUserContent.FORMAT_KEY);
 		return new AgentRunRequest(parentRoomId, continuationInput, engineIdFallback, harnessType, workspaceId,
 				maxTurns, maxReflections, paramMap, continuationAgentParams, null, null, continuationInsight, false, null,
 				SubAgentRunCompletionMode.WAIT, ownerAuthType, childRunId, continuationDepth + 1, null);

src/prerna/reactor/agent/run/AgentRunService.java

@@ -152,6 +152,7 @@ public AgentRunHandle runWithId(String runId, AgentRunRequest request) {
 	}
 
 	private AgentRunHandle submitNewRun(String runId, AgentRunRequest request, boolean ownsUser) {
+		request = request.withNormalizedUserContent();
 		String userId = resolveUserId(request.getInsight());
 		// A guardrail, the room's user or an admin closed it; no agent runs in it
 		// again.

src/prerna/reactor/agent/runtime/SemossAgentHarness.java

@@ -57,6 +57,7 @@
 import prerna.engine.impl.model.responses.AskErrorModelEngineResponse;
 import prerna.om.Insight;
 import prerna.om.ThreadStore;
+import prerna.reactor.agent.AgentUserContent;
 import prerna.reactor.agent.AgentHarnessResult;
 import prerna.reactor.agent.AgentRunContext;
 import prerna.reactor.agent.AgentRunner;
@@ -368,6 +369,7 @@ public AgentHarnessResult execute(AgentRunContext ctx) throws Exception {
 						.withText(ctx.getInput() + priorRequests + "\n\n" + state.runtimeContext(), ctx.getUserText())
 						.withMediaInputs(ctx.getMediaInputPaths(), room).withMediaUrls(ctx.getMediaUrls())
 						.withModelType(ctx.getModelEngine().getModelType()).withParamMap(paramMap).build();
+				AgentUserContent.annotate(firstMsg, ctx.getAgentConfig().getAgentParams());
 				tagAgentRun(firstMsg, ctx.getRunId(), RUN_ROLE_INPUT);
 				// A continuation prompt is platform-authored model context, not something the
 				// user said.