From bbd354a3500222e412e543dc74de95993f2fca24 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 19:58:11 +0200 Subject: [PATCH 1/2] feat: add bounded char sequence LTS execution --- .../runtime/LinearTokenSequenceMatcher.java | 248 ++++++++++++------ .../LinearTokenSequenceAccessLogTest.java | 108 ++++++++ .../LinearTokenSequenceMatcherTest.java | 179 +++++++++++++ 3 files changed, 449 insertions(+), 86 deletions(-) diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java index 43d985f8..a945317c 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java @@ -46,7 +46,7 @@ boolean embedsNameMap() { @Override public boolean matches(String input) { Objects.requireNonNull(input, "input"); - return matchesAt(input, 0, newWorkspace(), true); + return matchesAt(input, 0, input.length(), newWorkspace(), true); } @Override @@ -60,7 +60,7 @@ public int findFrom(String input, int start) { if (start < 0 || start > input.length()) return -1; MatchWorkspace workspace = newWorkspace(); for (int pos = start; pos <= input.length(); pos++) { - if (matchesAt(input, pos, workspace, false)) return pos; + if (matchesAt(input, pos, input.length(), workspace, false)) return pos; } return -1; } @@ -69,28 +69,24 @@ public int findFrom(String input, int start) { public MatchResult match(String input) { Objects.requireNonNull(input, "input"); MatchWorkspace workspace = newWorkspace(); - if (!matchesAt(input, 0, workspace, true)) return null; - if (!nameToIndex.isEmpty()) { - return new NamedMatchResultImpl( - input, workspace.starts, workspace.ends, groupCount, nameToIndex); - } - return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex); + if (!matchesAt(input, 0, input.length(), workspace, true)) return null; + return toMatchResult(input, workspace); } @Override public boolean matchesBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); - return start >= 0 - && end >= start - && end <= input.length() - && matches(input.subSequence(start, end).toString()); + if (!isValidRegion(input, start, end)) return false; + return matchesAt(input, start, end, newWorkspace(), true); } @Override public MatchResult matchBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); - if (start < 0 || end < start || end > input.length()) return null; - return match(input.subSequence(start, end).toString()); + if (!isValidRegion(input, start, end)) return null; + MatchWorkspace workspace = newWorkspace(); + if (!matchesAt(input, start, end, workspace, true)) return null; + return toMatchResult(input.toString(), workspace); } @Override @@ -104,38 +100,59 @@ public MatchResult findMatchFrom(String input, int start) { if (start < 0 || start > input.length()) return null; MatchWorkspace workspace = newWorkspace(); for (int pos = start; pos <= input.length(); pos++) { - if (!matchesAt(input, pos, workspace, false)) { + if (!matchesAt(input, pos, input.length(), workspace, false)) { continue; } - if (!nameToIndex.isEmpty()) { - return new NamedMatchResultImpl( - input, workspace.starts, workspace.ends, groupCount, nameToIndex); - } - return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex); + return toMatchResult(input, workspace); } return null; } @Override public boolean matchInto(String input, int[] groupStarts, int[] groupEnds) { + return matchIntoBounded(input, 0, input.length(), groupStarts, groupEnds); + } + + boolean matchIntoBounded( + CharSequence input, int start, int end, int[] groupStarts, int[] groupEnds) { Objects.requireNonNull(input, "input"); Objects.requireNonNull(groupStarts, "groupStarts"); Objects.requireNonNull(groupEnds, "groupEnds"); + validateRegion(input, start, end); if (groupStarts.length <= groupCount || groupEnds.length <= groupCount) { throw new IndexOutOfBoundsException("group arrays too small for " + groupCount + " groups"); } MatchWorkspace workspace = newWorkspace(); - if (!matchesAt(input, 0, workspace, true)) return false; + if (!matchesAt(input, start, end, workspace, true)) return false; System.arraycopy(workspace.starts, 0, groupStarts, 0, groupCount + 1); System.arraycopy(workspace.ends, 0, groupEnds, 0, groupCount + 1); return true; } + private MatchResult toMatchResult(String input, MatchWorkspace workspace) { + if (!nameToIndex.isEmpty()) { + return new NamedMatchResultImpl( + input, workspace.starts, workspace.ends, groupCount, nameToIndex); + } + return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex); + } + + private static void validateRegion(CharSequence input, int start, int end) { + if (!isValidRegion(input, start, end)) { + throw new IndexOutOfBoundsException("invalid region [" + start + ", " + end + ")"); + } + } + + private static boolean isValidRegion(CharSequence input, int start, int end) { + return start >= 0 && end >= start && end <= input.length(); + } + private MatchWorkspace newWorkspace() { return new MatchWorkspace(groupCount, optionalDepth); } - private boolean matchesAt(String input, int offset, MatchWorkspace workspace, boolean fullMatch) { + private boolean matchesAt( + CharSequence input, int offset, int regionEnd, MatchWorkspace workspace, boolean fullMatch) { int[] starts = workspace.starts; int[] ends = workspace.ends; Arrays.fill(starts, -1); @@ -147,109 +164,130 @@ private boolean matchesAt(String input, int offset, MatchWorkspace workspace, bo if (isTargetBeforeOptionalHttpVersion(plan.ops(), i)) { pos = captureTargetBeforeOptionalHttpVersion( - op, plan.ops().get(i + 1), input, pos, starts, ends); + op, plan.ops().get(i + 1), input, pos, regionEnd, starts, ends); if (pos < 0) return false; i++; continue; } - pos = apply(op, input, pos, starts, ends, i == plan.ops().size() - 1, workspace, 0); + pos = + apply(op, input, pos, regionEnd, starts, ends, i == plan.ops().size() - 1, workspace, 0); if (pos < 0) return false; } - if (fullMatch && pos != input.length()) return false; - ends[0] = fullMatch ? input.length() : pos; + if (fullMatch && pos != regionEnd) return false; + ends[0] = fullMatch ? regionEnd : pos; return true; } private int apply( LinearTokenSequencePlan.Op op, - String input, + CharSequence input, int pos, + int regionEnd, int[] starts, int[] ends, boolean lastOp, MatchWorkspace workspace, int optionalDepth) { return switch (op.kind()) { - case LITERAL -> startsWith(input, pos, op.literal()) ? pos + op.literal().length() : -1; - case WHITESPACE_PLUS -> skipWhitespace(input, pos); - case CAPTURE_NON_SPACE -> captureNonSpace(input, pos, op.groupNumber(), starts, ends); - case CAPTURE_DIGITS -> captureDigits(input, pos, op.groupNumber(), starts, ends); + case LITERAL -> + startsWith(input, pos, regionEnd, op.literal()) ? pos + op.literal().length() : -1; + case WHITESPACE_PLUS -> skipWhitespace(input, pos, regionEnd); + case CAPTURE_NON_SPACE -> + captureNonSpace(input, pos, regionEnd, op.groupNumber(), starts, ends); + case CAPTURE_DIGITS -> captureDigits(input, pos, regionEnd, op.groupNumber(), starts, ends); case CAPTURE_SIGNED_INTEGER -> - captureSignedInteger(input, pos, op.groupNumber(), starts, ends); + captureSignedInteger(input, pos, regionEnd, op.groupNumber(), starts, ends); case CAPTURE_SIGNED_INTEGER_OR_DASH -> - captureSignedIntegerOrDash(input, pos, op.groupNumber(), starts, ends, true); + captureSignedIntegerOrDash(input, pos, regionEnd, op.groupNumber(), starts, ends, true); case CAPTURE_SIGNED_INTEGER_OR_UNCAPTURED_DASH -> - captureSignedIntegerOrDash(input, pos, op.groupNumber(), starts, ends, false); + captureSignedIntegerOrDash(input, pos, regionEnd, op.groupNumber(), starts, ends, false); case CAPTURE_DECIMAL_NUMBER -> - captureDecimal(input, pos, op.groupNumber(), starts, ends, false); + captureDecimal(input, pos, regionEnd, op.groupNumber(), starts, ends, false); case CAPTURE_SIGNED_DECIMAL_NUMBER -> - captureDecimal(input, pos, op.groupNumber(), starts, ends, true); - case CAPTURE_WORD -> captureWord(input, pos, op.groupNumber(), starts, ends); + captureDecimal(input, pos, regionEnd, op.groupNumber(), starts, ends, true); + case CAPTURE_WORD -> captureWord(input, pos, regionEnd, op.groupNumber(), starts, ends); case CAPTURE_UNTIL_DELIMITER -> - captureUntil(input, pos, op.delimiter(), op.groupNumber(), starts, ends); + captureUntil(input, pos, regionEnd, op.delimiter(), op.groupNumber(), starts, ends); case CAPTURE_QUOTED_UNTIL_DELIMITER -> - captureQuotedUntil(input, pos, op.delimiter(), op.groupNumber(), starts, ends, false); + captureQuotedUntil( + input, pos, regionEnd, op.delimiter(), op.groupNumber(), starts, ends, false); case CAPTURE_QUOTED_NON_SPACE -> - captureQuotedUntil(input, pos, op.delimiter(), op.groupNumber(), starts, ends, true); - case CAPTURE_IP_OR_HOST -> captureIpOrHost(input, pos, op.groupNumber(), starts, ends); + captureQuotedUntil( + input, pos, regionEnd, op.delimiter(), op.groupNumber(), starts, ends, true); + case CAPTURE_IP_OR_HOST -> + captureIpOrHost(input, pos, regionEnd, op.groupNumber(), starts, ends); case CAPTURE_BRACKETED_WORD_AFTER_SKIP -> - captureBracketedWordAfterSkip(input, pos, op.groupNumber(), starts, ends); - case SKIP_ANY -> lastOp ? input.length() : -1; + captureBracketedWordAfterSkip(input, pos, regionEnd, op.groupNumber(), starts, ends); + case SKIP_ANY -> lastOp ? consumeToEnd(input, pos, regionEnd) : -1; case ANCHOR -> pos; case OPTIONAL_SEQUENCE -> - applyOptional(op, input, pos, starts, ends, workspace, optionalDepth); + applyOptional(op, input, pos, regionEnd, starts, ends, workspace, optionalDepth); }; } - private static int captureNonSpace(String input, int pos, int group, int[] starts, int[] ends) { + private static int captureNonSpace( + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int start = pos; - while (pos < input.length() && !Character.isWhitespace(input.charAt(pos))) pos++; + while (pos < regionEnd && !Character.isWhitespace(input.charAt(pos))) pos++; if (pos == start) return -1; set(starts, ends, group, start, pos); return pos; } - private static int captureDigits(String input, int pos, int group, int[] starts, int[] ends) { + private static int captureDigits( + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int start = pos; - while (pos < input.length() && isDigit(input.charAt(pos))) pos++; + while (pos < regionEnd && isDigit(input.charAt(pos))) pos++; if (pos == start) return -1; set(starts, ends, group, start, pos); return pos; } private static int captureSignedInteger( - String input, int pos, int group, int[] starts, int[] ends) { + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int start = pos; - if (pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) pos++; + if (pos < regionEnd && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) pos++; int digitStart = pos; - while (pos < input.length() && isDigit(input.charAt(pos))) pos++; + while (pos < regionEnd && isDigit(input.charAt(pos))) pos++; if (pos == digitStart) return -1; set(starts, ends, group, start, pos); return pos; } private static int captureSignedIntegerOrDash( - String input, int pos, int group, int[] starts, int[] ends, boolean captureDash) { - if (pos < input.length() && input.charAt(pos) == '-') { + CharSequence input, + int pos, + int regionEnd, + int group, + int[] starts, + int[] ends, + boolean captureDash) { + if (pos < regionEnd && input.charAt(pos) == '-') { if (captureDash) set(starts, ends, group, pos, pos + 1); return pos + 1; } - return captureSignedInteger(input, pos, group, starts, ends); + return captureSignedInteger(input, pos, regionEnd, group, starts, ends); } private static int captureDecimal( - String input, int pos, int group, int[] starts, int[] ends, boolean signed) { + CharSequence input, + int pos, + int regionEnd, + int group, + int[] starts, + int[] ends, + boolean signed) { int start = pos; - if (signed && pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { + if (signed && pos < regionEnd && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { pos++; } int digitStart = pos; - while (pos < input.length() && isDigit(input.charAt(pos))) pos++; + while (pos < regionEnd && isDigit(input.charAt(pos))) pos++; boolean sawLeadingDigits = pos > digitStart; - if (pos < input.length() && input.charAt(pos) == '.') { + if (pos < regionEnd && input.charAt(pos) == '.') { pos++; int fractionStart = pos; - while (pos < input.length() && isDigit(input.charAt(pos))) pos++; + while (pos < regionEnd && isDigit(input.charAt(pos))) pos++; if (!sawLeadingDigits && pos == fractionStart) return -1; } else if (!sawLeadingDigits) { return -1; @@ -258,34 +296,42 @@ private static int captureDecimal( return pos; } - private static int captureWord(String input, int pos, int group, int[] starts, int[] ends) { + private static int captureWord( + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int start = pos; - while (pos < input.length() && isWord(input.charAt(pos))) pos++; + while (pos < regionEnd && isWord(input.charAt(pos))) pos++; if (pos == start) return -1; set(starts, ends, group, start, pos); return pos; } private static int captureUntil( - String input, int pos, char delimiter, int group, int[] starts, int[] ends) { - int end = input.indexOf(delimiter, pos); - if (end < 0) return -1; + CharSequence input, + int pos, + int regionEnd, + char delimiter, + int group, + int[] starts, + int[] ends) { + int end = findChar(input, pos, regionEnd, delimiter); + if (end == regionEnd) return -1; set(starts, ends, group, pos, end); return end; } private static int captureQuotedUntil( - String input, + CharSequence input, int pos, + int regionEnd, char delimiter, int group, int[] starts, int[] ends, boolean nonSpace) { - if (pos >= input.length() || input.charAt(pos) != '"') return -1; + if (pos >= regionEnd || input.charAt(pos) != '"') return -1; int start = pos + 1; - int end = input.indexOf(delimiter, start); - if (end < 0) return -1; + int end = findChar(input, start, regionEnd, delimiter); + if (end == regionEnd) return -1; if (nonSpace) { for (int i = start; i < end; i++) { if (Character.isWhitespace(input.charAt(i))) return -1; @@ -295,18 +341,19 @@ private static int captureQuotedUntil( return end + 1; } - private static int captureIpOrHost(String input, int pos, int group, int[] starts, int[] ends) { - int end = captureNonSpace(input, pos, group, starts, ends); + private static int captureIpOrHost( + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { + int end = captureNonSpace(input, pos, regionEnd, group, starts, ends); return end >= 0 && isIpOrHost(input, pos, end) ? end : -1; } private static int captureBracketedWordAfterSkip( - String input, int pos, int group, int[] starts, int[] ends) { + CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int lastStart = -1; int lastEnd = -1; int open = -1; int wordEnd = -1; - for (int index = pos; index < input.length(); index++) { + for (int index = pos; index < regionEnd; index++) { char ch = input.charAt(index); if (ch == '[') { open = index; @@ -319,7 +366,7 @@ private static int captureBracketedWordAfterSkip( if (ch == ']') { if (wordEnd == index && wordEnd > open + 1 - && index + 1 < input.length() + && index + 1 < regionEnd && Character.isWhitespace(input.charAt(index + 1))) { lastStart = open + 1; lastEnd = index; @@ -334,7 +381,7 @@ private static int captureBracketedWordAfterSkip( } if (lastStart < 0) return -1; set(starts, ends, group, lastStart, lastEnd); - return input.length(); + return regionEnd; } private static boolean isTargetBeforeOptionalHttpVersion( @@ -358,13 +405,14 @@ private static boolean isTargetBeforeOptionalHttpVersion( private static int captureTargetBeforeOptionalHttpVersion( LinearTokenSequencePlan.Op target, LinearTokenSequencePlan.Op optionalHttpVersion, - String input, + CharSequence input, int pos, + int regionEnd, int[] starts, int[] ends) { - int quote = input.indexOf('"', pos); - if (quote < 0 || quote == pos) return -1; - int marker = input.lastIndexOf(" HTTP/", quote); + int quote = findChar(input, pos, regionEnd, '"'); + if (quote == regionEnd || quote == pos) return -1; + int marker = findLastLiteral(input, pos, quote, " HTTP/"); if (marker >= pos && isNonSpace(input, pos, marker)) { LinearTokenSequencePlan.Op version = optionalHttpVersion.children().get(1); int versionStart = marker + " HTTP/".length(); @@ -382,8 +430,9 @@ private static int captureTargetBeforeOptionalHttpVersion( private int applyOptional( LinearTokenSequencePlan.Op op, - String input, + CharSequence input, int pos, + int regionEnd, int[] starts, int[] ends, MatchWorkspace workspace, @@ -399,6 +448,7 @@ private int applyOptional( op.children().get(i), input, next, + regionEnd, starts, ends, i == op.children().size() - 1, @@ -441,14 +491,40 @@ private static final class MatchWorkspace { } } - private static int skipWhitespace(String input, int pos) { + private static int skipWhitespace(CharSequence input, int pos, int regionEnd) { int start = pos; - while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) pos++; + while (pos < regionEnd && Character.isWhitespace(input.charAt(pos))) pos++; return pos == start ? -1 : pos; } - private static boolean startsWith(String input, int pos, String prefix) { - return pos >= 0 && pos + prefix.length() <= input.length() && input.startsWith(prefix, pos); + private static boolean startsWith(CharSequence input, int pos, int regionEnd, String prefix) { + if (pos < 0 || pos + prefix.length() > regionEnd) return false; + for (int i = 0; i < prefix.length(); i++) { + if (input.charAt(pos + i) != prefix.charAt(i)) return false; + } + return true; + } + + private static int findChar(CharSequence input, int pos, int regionEnd, char target) { + for (int i = pos; i < regionEnd; i++) { + if (input.charAt(i) == target) return i; + } + return regionEnd; + } + + private static int findLastLiteral(CharSequence input, int start, int end, String literal) { + int last = -1; + for (int pos = start; pos + literal.length() <= end; pos++) { + if (startsWith(input, pos, end, literal)) last = pos; + } + return last; + } + + private static int consumeToEnd(CharSequence input, int pos, int regionEnd) { + while (pos < regionEnd) { + input.charAt(pos++); + } + return regionEnd; } private static void set(int[] starts, int[] ends, int group, int start, int end) { @@ -458,7 +534,7 @@ private static void set(int[] starts, int[] ends, int group, int start, int end) } } - private static boolean isIpOrHost(String input, int start, int end) { + private static boolean isIpOrHost(CharSequence input, int start, int end) { for (int i = start; i < end; i++) { char ch = input.charAt(i); if (!isAsciiAlphaNum(ch) && ch != '-' && ch != '_' && ch != '.' && ch != ':' && ch != '%') { @@ -468,7 +544,7 @@ private static boolean isIpOrHost(String input, int start, int end) { return end > start; } - private static boolean isNonSpace(String input, int start, int end) { + private static boolean isNonSpace(CharSequence input, int start, int end) { if (end <= start) return false; for (int i = start; i < end; i++) { if (Character.isWhitespace(input.charAt(i))) return false; @@ -476,7 +552,7 @@ private static boolean isNonSpace(String input, int start, int end) { return true; } - private static int scanDecimal(String input, int pos, int limit, boolean signed) { + private static int scanDecimal(CharSequence input, int pos, int limit, boolean signed) { if (signed && pos < limit && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { pos++; } diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java index 9cf193b7..51ca8ed1 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java @@ -173,6 +173,42 @@ void realExpandedCombinedPatternHasJdkEquivalentNamedCaptureBoundaries() throws "[not_the_logger] . [nginx_access] [host-with-dash]")); } + @Test + void realExpandedPatternsUseOnlyTheBoundedCharSequenceRegion() throws Exception { + assertBoundedFixtureUsesOnlyItsRegion( + testResource("logs-grok-pattern-1.regex"), + commonMessage("10.202.82.195", "POST ", "/config?x=y", " HTTP/1.1", "17888"), + commonMessage("10.202.82.195", "POST ", "/config?x=y", " HTTP/1.1", "not-a-number")); + assertBoundedFixtureUsesOnlyItsRegion( + testResource("logs-grok-pattern-2.regex"), + combinedMessage( + "10.202.82.195", + "POST ", + "/config?x=y", + " HTTP/1.1", + "17888", + "https://example.com/index.html", + "Mozilla/5.0 Test", + "-", + "tracking-id", + "0.024", + "0.024", + "[decoy] . [nginx_access] [not-the-logger]"), + combinedMessage( + "10.202.82.195", + "POST ", + "/config?x=y", + " HTTP/1.1", + "not-a-number", + "https://example.com/index.html", + "Mozilla/5.0 Test", + "-", + "tracking-id", + "0.024", + "0.024", + "[decoy] . [nginx_access] [not-the-logger]")); + } + @Test void leavesCallerArraysUnchangedOnNoMatch() { ReggieMatcher matcher = Reggie.compile(COMBINED_ACCESS_LOG_PATTERN, NAMED_ONLY); @@ -232,6 +268,43 @@ private static void assertNamedCaptureBoundariesEquivalent(String pattern, Strin } } + private static void assertBoundedFixtureUsesOnlyItsRegion( + String pattern, String matchingInput, String failingInput) throws Exception { + ReggieMatcher matcher = Reggie.compile(pattern, NAMED_ONLY); + assertDelegateType(matcher, LinearTokenSequenceMatcher.class); + LinearTokenSequenceMatcher ltsMatcher = (LinearTokenSequenceMatcher) matcher; + int groupCount = Pattern.compile(pattern).matcher("").groupCount(); + + assertBoundedResult( + ltsMatcher, groupCount, "before " + matchingInput + " after ", matchingInput, true); + assertBoundedResult( + ltsMatcher, groupCount, "before " + failingInput + " after ", failingInput, false); + } + + private static void assertBoundedResult( + LinearTokenSequenceMatcher matcher, + int groupCount, + String source, + String region, + boolean expectedMatch) { + int start = source.indexOf(region); + int end = start + region.length(); + RangeGuardCharSequence input = new RangeGuardCharSequence(source, start, end); + int[] starts = new int[groupCount + 1]; + int[] ends = new int[groupCount + 1]; + Arrays.fill(starts, 777); + Arrays.fill(ends, 888); + + assertEquals(expectedMatch, matcher.matchIntoBounded(input, start, end, starts, ends)); + if (expectedMatch) { + assertEquals(start, starts[0]); + assertEquals(end, ends[0]); + } else { + assertTrue(Arrays.stream(starts).allMatch(value -> value == 777)); + assertTrue(Arrays.stream(ends).allMatch(value -> value == 888)); + } + } + private static String commonMessage( String client, String methodWithSpace, @@ -302,4 +375,39 @@ private static void assertDelegateTypeUnchecked(ReggieMatcher matcher, Class throw new AssertionError(e); } } + + private static final class RangeGuardCharSequence implements CharSequence { + private final String value; + private final int start; + private final int end; + + RangeGuardCharSequence(String value, int start, int end) { + this.value = value; + this.start = start; + this.end = end; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + if (index < start || index >= end) { + throw new AssertionError("read outside bounded region: " + index); + } + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new AssertionError("subSequence must not be called while matching"); + } + + @Override + public String toString() { + throw new AssertionError("toString must not be called while matching"); + } + } } diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java index 07578216..e498f8bf 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java @@ -187,6 +187,105 @@ void namedOnlyRoutingRejectsPlansMissingAnObservableNamedCapture() throws Except } } + @Test + void boundedExecutionUsesOnlyCharAtAndReturnsAbsoluteSpans() throws Exception { + LinearTokenSequenceMatcher matcher = + (LinearTokenSequenceMatcher) matcherFor("host=(?\\S+) status=(?[+-]?\\d+)"); + String source = "prefix host=api.example status=200 suffix"; + int start = source.indexOf("host="); + int end = source.indexOf(" suffix"); + RangeGuardCharSequence input = new RangeGuardCharSequence(source, start, end); + int[] starts = new int[3]; + int[] ends = new int[3]; + + assertTrue(matcher.matchIntoBounded(input, start, end, starts, ends)); + + assertEquals(start, starts[0]); + assertEquals(end, ends[0]); + assertEquals("api.example", source.substring(starts[1], ends[1])); + assertEquals("200", source.substring(starts[2], ends[2])); + } + + @Test + void boundedExecutionRejectsInvalidArgumentsAndKeepsArraysAtomicOnFailure() throws Exception { + LinearTokenSequenceMatcher matcher = + (LinearTokenSequenceMatcher) matcherFor("host=(?\\S+) status=(?[+-]?\\d+)"); + int[] starts = new int[] {7, 7, 7}; + int[] ends = new int[] {9, 9, 9}; + CharSequence input = new ConversionFailingCharSequence("host=api status=not-a-number"); + + assertFalse(matcher.matchIntoBounded(input, 0, input.length(), starts, ends)); + assertTrue(Arrays.equals(new int[] {7, 7, 7}, starts)); + assertTrue(Arrays.equals(new int[] {9, 9, 9}, ends)); + assertThrows( + NullPointerException.class, + () -> matcher.matchIntoBounded(null, 0, 0, new int[3], new int[3])); + assertThrows( + NullPointerException.class, + () -> matcher.matchIntoBounded(input, 0, input.length(), null, new int[3])); + assertThrows( + IndexOutOfBoundsException.class, + () -> matcher.matchIntoBounded(input, -1, input.length(), new int[3], new int[3])); + assertThrows( + IndexOutOfBoundsException.class, + () -> matcher.matchIntoBounded(input, 0, input.length(), new int[2], new int[3])); + } + + @Test + void boundedExecutionCoversQuotedAndOptionalHttpVersionPathsWithoutConversion() throws Exception { + LinearTokenSequenceMatcher quoted = + (LinearTokenSequenceMatcher) matcherFor("referer=\"(?[^\"]*)\""); + String quotedSource = "referer=\"https://example.com/index.html\""; + CharSequence quotedInput = new ConversionFailingCharSequence(quotedSource); + assertTrue(quoted.matchesBounded(quotedInput, 0, quotedInput.length())); + + LinearTokenSequenceMatcher request = + (LinearTokenSequenceMatcher) + matcherFor( + "\"(?\\b\\w+\\b) (?\\S+)(?: HTTP/(?\\d+\\.\\d+)|)\""); + String requestSource = "\"GET /health HTTP/2.0\""; + CharSequence requestInput = new ConversionFailingCharSequence(requestSource); + int[] starts = new int[4]; + int[] ends = new int[4]; + assertTrue(request.matchIntoBounded(requestInput, 0, requestInput.length(), starts, ends)); + assertEquals("GET", requestSource.substring(starts[1], ends[1])); + assertEquals("/health", requestSource.substring(starts[2], ends[2])); + assertEquals("2.0", requestSource.substring(starts[3], ends[3])); + } + + @Test + void finalSkipAnyConsumesTheTailAndPropagatesCharAtFailures() throws Exception { + LinearTokenSequenceMatcher matcher = + (LinearTokenSequenceMatcher) matcherFor("(?\\S+) .*"); + String source = "token remaining"; + CountingCharSequence counted = new CountingCharSequence(source); + + assertTrue(matcher.matchesBounded(counted, 0, counted.length())); + assertTrue(counted.charAtCalls >= source.length()); + assertThrows( + CharAtFailure.class, + () -> + matcher.matchesBounded( + new FailingAtCharSequence(source, source.indexOf('r')), 0, source.length())); + } + + @Test + void matchBoundedMaterializesOnlyAfterSuccessAndUsesAbsoluteSpans() throws Exception { + LinearTokenSequenceMatcher matcher = + (LinearTokenSequenceMatcher) matcherFor("(?\\S+)(?: (?\\d+)|)", 2); + CharSequence failing = new ConversionFailingCharSequence("host not-a-number"); + assertNull(matcher.matchBounded(failing, 0, failing.length())); + + String source = "xxapiyy"; + MatchResult result = matcher.matchBounded(source, 2, 5); + assertNotNull(result); + assertEquals(2, result.start()); + assertEquals(5, result.end()); + assertEquals("api", result.group("host")); + assertEquals(-1, result.start("status")); + assertNull(result.group("status")); + } + @Test void capturedDashAlternativeRecordsNamedGroupSpan() throws Exception { ReggieMatcher matcher = Reggie.compile("(?(?:-|[+-]?\\d+))", NAMED_ONLY_OPTIONS); @@ -317,4 +416,84 @@ private static ReggieMatcher matcherFor(String pattern, int groupCount) throws E LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)).orElseThrow(); return new LinearTokenSequenceMatcher(pattern, plan, groupCount, names); } + + private static class ConversionFailingCharSequence implements CharSequence { + final String value; + + ConversionFailingCharSequence(String value) { + this.value = value; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new AssertionError("subSequence must not be called while matching"); + } + + @Override + public String toString() { + throw new AssertionError("toString must not be called while matching"); + } + } + + private static final class RangeGuardCharSequence extends ConversionFailingCharSequence { + private final int allowedStart; + private final int allowedEnd; + + RangeGuardCharSequence(String value, int allowedStart, int allowedEnd) { + super(value); + this.allowedStart = allowedStart; + this.allowedEnd = allowedEnd; + } + + @Override + public char charAt(int index) { + if (index < allowedStart || index >= allowedEnd) { + throw new AssertionError("read outside bounded region: " + index); + } + return super.charAt(index); + } + } + + private static final class CountingCharSequence extends ConversionFailingCharSequence { + int charAtCalls; + + CountingCharSequence(String value) { + super(value); + } + + @Override + public char charAt(int index) { + charAtCalls++; + return super.charAt(index); + } + } + + private static final class FailingAtCharSequence extends ConversionFailingCharSequence { + private final int failingIndex; + + FailingAtCharSequence(String value, int failingIndex) { + super(value); + this.failingIndex = failingIndex; + } + + @Override + public char charAt(int index) { + if (index == failingIndex) { + throw new CharAtFailure(); + } + return super.charAt(index); + } + } + + private static final class CharAtFailure extends RuntimeException {} } From 162779e9d03a2d054c4748ea17447e42791c88e6 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 20 Jul 2026 17:31:45 +0200 Subject: [PATCH 2/2] fix: pool LTS match workspace per-thread, avoid full materialization in matchBounded Restores the allocation-free contract on LinearTokenSequenceMatcher via a ThreadLocal workspace (defensively cloned into MatchResult to avoid aliasing across reuse), fixes matchBounded to materialize only [0, end), adds String fast paths to internal scans, and dedupes the RangeGuardCharSequence test double. Co-Authored-By: Claude Sonnet 5 --- .../runtime/LinearTokenSequenceMatcher.java | 51 +++++++---- .../LinearTokenSequenceAccessLogTest.java | 35 -------- .../LinearTokenSequenceMatcherTest.java | 84 ++++++++++++++----- .../runtime/RangeGuardCharSequence.java | 52 ++++++++++++ 4 files changed, 153 insertions(+), 69 deletions(-) create mode 100644 reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java index a945317c..6964648e 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java @@ -25,6 +25,7 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { private final LinearTokenSequencePlan plan; private final int groupCount; private final int optionalDepth; + private final ThreadLocal workspaceHolder; LinearTokenSequenceMatcher( String pattern, @@ -36,6 +37,8 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { this.groupCount = groupCount; this.nameToIndex = Map.copyOf(nameToIndex); this.optionalDepth = maxOptionalDepth(plan.ops()); + this.workspaceHolder = + ThreadLocal.withInitial(() -> new MatchWorkspace(this.groupCount, this.optionalDepth)); } @Override @@ -46,7 +49,7 @@ boolean embedsNameMap() { @Override public boolean matches(String input) { Objects.requireNonNull(input, "input"); - return matchesAt(input, 0, input.length(), newWorkspace(), true); + return matchesAt(input, 0, input.length(), workspaceHolder.get(), true); } @Override @@ -58,7 +61,7 @@ public boolean find(String input) { public int findFrom(String input, int start) { Objects.requireNonNull(input, "input"); if (start < 0 || start > input.length()) return -1; - MatchWorkspace workspace = newWorkspace(); + MatchWorkspace workspace = workspaceHolder.get(); for (int pos = start; pos <= input.length(); pos++) { if (matchesAt(input, pos, input.length(), workspace, false)) return pos; } @@ -68,7 +71,7 @@ public int findFrom(String input, int start) { @Override public MatchResult match(String input) { Objects.requireNonNull(input, "input"); - MatchWorkspace workspace = newWorkspace(); + MatchWorkspace workspace = workspaceHolder.get(); if (!matchesAt(input, 0, input.length(), workspace, true)) return null; return toMatchResult(input, workspace); } @@ -77,16 +80,16 @@ public MatchResult match(String input) { public boolean matchesBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); if (!isValidRegion(input, start, end)) return false; - return matchesAt(input, start, end, newWorkspace(), true); + return matchesAt(input, start, end, workspaceHolder.get(), true); } @Override public MatchResult matchBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); if (!isValidRegion(input, start, end)) return null; - MatchWorkspace workspace = newWorkspace(); + MatchWorkspace workspace = workspaceHolder.get(); if (!matchesAt(input, start, end, workspace, true)) return null; - return toMatchResult(input.toString(), workspace); + return toMatchResult(input.subSequence(0, end).toString(), workspace); } @Override @@ -98,7 +101,7 @@ public MatchResult findMatch(String input) { public MatchResult findMatchFrom(String input, int start) { Objects.requireNonNull(input, "input"); if (start < 0 || start > input.length()) return null; - MatchWorkspace workspace = newWorkspace(); + MatchWorkspace workspace = workspaceHolder.get(); for (int pos = start; pos <= input.length(); pos++) { if (!matchesAt(input, pos, input.length(), workspace, false)) { continue; @@ -122,7 +125,7 @@ boolean matchIntoBounded( if (groupStarts.length <= groupCount || groupEnds.length <= groupCount) { throw new IndexOutOfBoundsException("group arrays too small for " + groupCount + " groups"); } - MatchWorkspace workspace = newWorkspace(); + MatchWorkspace workspace = workspaceHolder.get(); if (!matchesAt(input, start, end, workspace, true)) return false; System.arraycopy(workspace.starts, 0, groupStarts, 0, groupCount + 1); System.arraycopy(workspace.ends, 0, groupEnds, 0, groupCount + 1); @@ -130,11 +133,14 @@ boolean matchIntoBounded( } private MatchResult toMatchResult(String input, MatchWorkspace workspace) { + // workspace is reused across calls on the same thread; MatchResultImpl/NamedMatchResultImpl + // hold onto the arrays they're given, so a defensive copy is required here. + int[] starts = Arrays.copyOf(workspace.starts, workspace.starts.length); + int[] ends = Arrays.copyOf(workspace.ends, workspace.ends.length); if (!nameToIndex.isEmpty()) { - return new NamedMatchResultImpl( - input, workspace.starts, workspace.ends, groupCount, nameToIndex); + return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex); } - return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex); + return new MatchResultImpl(input, starts, ends, groupCount, nameToIndex); } private static void validateRegion(CharSequence input, int start, int end) { @@ -147,10 +153,6 @@ private static boolean isValidRegion(CharSequence input, int start, int end) { return start >= 0 && end >= start && end <= input.length(); } - private MatchWorkspace newWorkspace() { - return new MatchWorkspace(groupCount, optionalDepth); - } - private boolean matchesAt( CharSequence input, int offset, int regionEnd, MatchWorkspace workspace, boolean fullMatch) { int[] starts = workspace.starts; @@ -499,6 +501,9 @@ private static int skipWhitespace(CharSequence input, int pos, int regionEnd) { private static boolean startsWith(CharSequence input, int pos, int regionEnd, String prefix) { if (pos < 0 || pos + prefix.length() > regionEnd) return false; + if (input instanceof String s) { + return s.startsWith(prefix, pos); + } for (int i = 0; i < prefix.length(); i++) { if (input.charAt(pos + i) != prefix.charAt(i)) return false; } @@ -506,6 +511,10 @@ private static boolean startsWith(CharSequence input, int pos, int regionEnd, St } private static int findChar(CharSequence input, int pos, int regionEnd, char target) { + if (input instanceof String s) { + int idx = s.indexOf(target, pos); + return idx >= 0 && idx < regionEnd ? idx : regionEnd; + } for (int i = pos; i < regionEnd; i++) { if (input.charAt(i) == target) return i; } @@ -513,6 +522,12 @@ private static int findChar(CharSequence input, int pos, int regionEnd, char tar } private static int findLastLiteral(CharSequence input, int start, int end, String literal) { + if (input instanceof String s) { + int fromIndex = end - literal.length(); + if (fromIndex < start) return -1; + int idx = s.lastIndexOf(literal, fromIndex); + return idx >= start ? idx : -1; + } int last = -1; for (int pos = start; pos + literal.length() <= end; pos++) { if (startsWith(input, pos, end, literal)) last = pos; @@ -521,6 +536,12 @@ private static int findLastLiteral(CharSequence input, int start, int end, Strin } private static int consumeToEnd(CharSequence input, int pos, int regionEnd) { + // String#charAt has no observable side effects, so the validation walk below — which exists + // to surface CharSequence implementations that throw or misbehave on out-of-range access — + // is unnecessary overhead for the common String case. + if (input instanceof String) { + return regionEnd; + } while (pos < regionEnd) { input.charAt(pos++); } diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java index 51ca8ed1..e68ababb 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java @@ -375,39 +375,4 @@ private static void assertDelegateTypeUnchecked(ReggieMatcher matcher, Class throw new AssertionError(e); } } - - private static final class RangeGuardCharSequence implements CharSequence { - private final String value; - private final int start; - private final int end; - - RangeGuardCharSequence(String value, int start, int end) { - this.value = value; - this.start = start; - this.end = end; - } - - @Override - public int length() { - return value.length(); - } - - @Override - public char charAt(int index) { - if (index < start || index >= end) { - throw new AssertionError("read outside bounded region: " + index); - } - return value.charAt(index); - } - - @Override - public CharSequence subSequence(int start, int end) { - throw new AssertionError("subSequence must not be called while matching"); - } - - @Override - public String toString() { - throw new AssertionError("toString must not be called while matching"); - } - } } diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java index e498f8bf..e9c5c38d 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java @@ -61,6 +61,13 @@ void handlesQuotedDelimiterCaptures() throws Exception { assertEquals("https://example.com/index.html", result.group("referer")); } + @Test + void quotedDelimiterCaptureFailsWhenClosingDelimiterIsMissing() throws Exception { + ReggieMatcher matcher = matcherFor("referer=\"(?[^\"]*)\""); + + assertNull(matcher.match("referer=\"https://example.com/index.html")); + } + @Test void matchReturnsNullWhenSequenceDoesNotMatch() throws Exception { ReggieMatcher matcher = matcherFor("host=(?\\S+) status=(?[+-]?\\d+)"); @@ -286,6 +293,64 @@ void matchBoundedMaterializesOnlyAfterSuccessAndUsesAbsoluteSpans() throws Excep assertNull(result.group("status")); } + @Test + void matchBoundedMaterializesOnlyTheRequestedRegionOnSuccess() throws Exception { + LinearTokenSequenceMatcher matcher = + (LinearTokenSequenceMatcher) matcherFor("(?\\S+)(?: (?\\d+)|)", 2); + String source = "xxapiyy trailing garbage that must never be read or materialized"; + int end = 5; + CharSequence input = new BoundedRegionCharSequence(source, end); + + MatchResult result = matcher.matchBounded(input, 2, end); + + assertNotNull(result); + assertEquals(2, result.start()); + assertEquals(end, result.end()); + assertEquals("api", result.group("host")); + } + + /** + * CharSequence test double that only permits charAt within {@code [0, end)} and only permits + * subSequence/toString for exactly {@code [0, end)} — used to prove matchBounded materializes + * just the requested region rather than the entire backing sequence. + */ + private static final class BoundedRegionCharSequence implements CharSequence { + private final String value; + private final int end; + + BoundedRegionCharSequence(String value, int end) { + this.value = value; + this.end = end; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + if (index < 0 || index >= end) { + throw new AssertionError("read outside bounded region: " + index); + } + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int subEnd) { + if (start != 0 || subEnd != end) { + throw new AssertionError( + "subSequence outside bounded region: [" + start + ", " + subEnd + ")"); + } + return value.substring(start, subEnd); + } + + @Override + public String toString() { + throw new AssertionError("toString must not be called directly; use subSequence(0, end)"); + } + } + @Test void capturedDashAlternativeRecordsNamedGroupSpan() throws Exception { ReggieMatcher matcher = Reggie.compile("(?(?:-|[+-]?\\d+))", NAMED_ONLY_OPTIONS); @@ -445,25 +510,6 @@ public String toString() { } } - private static final class RangeGuardCharSequence extends ConversionFailingCharSequence { - private final int allowedStart; - private final int allowedEnd; - - RangeGuardCharSequence(String value, int allowedStart, int allowedEnd) { - super(value); - this.allowedStart = allowedStart; - this.allowedEnd = allowedEnd; - } - - @Override - public char charAt(int index) { - if (index < allowedStart || index >= allowedEnd) { - throw new AssertionError("read outside bounded region: " + index); - } - return super.charAt(index); - } - } - private static final class CountingCharSequence extends ConversionFailingCharSequence { int charAtCalls; diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java new file mode 100644 index 00000000..aa4b9070 --- /dev/null +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026-Present Datadog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.reggie.runtime; + +/** CharSequence test double that fails any charAt outside an allowed region. */ +final class RangeGuardCharSequence implements CharSequence { + private final String value; + private final int allowedStart; + private final int allowedEnd; + + RangeGuardCharSequence(String value, int allowedStart, int allowedEnd) { + this.value = value; + this.allowedStart = allowedStart; + this.allowedEnd = allowedEnd; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + if (index < allowedStart || index >= allowedEnd) { + throw new AssertionError("read outside bounded region: " + index); + } + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new AssertionError("subSequence must not be called while matching"); + } + + @Override + public String toString() { + throw new AssertionError("toString must not be called while matching"); + } +}