diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1bd8426afd..55f08c1eae 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/publish-documentation.yml b/.github/workflows/publish-documentation.yml index 33baea4572..2cab4fa232 100644 --- a/.github/workflows/publish-documentation.yml +++ b/.github/workflows/publish-documentation.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK uses: actions/setup-java@v4 with: diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 8d22f67352..ebd3929f17 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -7,14 +7,14 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '21' cache: 'maven' - server-id: ossrh + server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} @@ -22,6 +22,6 @@ jobs: - name: Publish snapshot run: ./mvnw clean deploy -Psnapshots -DskipITs -DskipTests env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb1c9f9029..0fd880b807 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,60 +8,33 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - name: Evaluate release type - run: ci/evaluate-release.sh + - uses: actions/checkout@v5 - name: Set up JDK uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' cache: 'maven' - server-id: ${{ env.maven_server_id }} + server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Release Stream Java Client (GA) - if: ${{ env.ga_release == 'true' }} + - name: Release Stream Java Client run: | git config user.name "rabbitmq-ci" git config user.email "rabbitmq-ci@users.noreply.github.com" ci/release-stream-java-client.sh env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} - - name: Release Stream Java Client (Milestone/RC) - if: ${{ env.ga_release != 'true' }} - run: | - git config user.name "rabbitmq-ci" - git config user.email "rabbitmq-ci@users.noreply.github.com" - ci/release-stream-java-client.sh - env: - MAVEN_USERNAME: '' - MAVEN_PASSWORD: ${{ secrets.PACKAGECLOUD_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} - - name: Checkout tls-gen - uses: actions/checkout@v4 - with: - repository: rabbitmq/tls-gen - path: './tls-gen' - - name: Start broker - run: ci/start-broker.sh - - name: Set up JDK for sanity check and documentation generation + - name: Set up JDK for documentation generation uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '21' cache: 'maven' - - name: Sanity Check - run: | - source ./release-versions.txt - export RABBITMQ_LIBRARY_VERSION=$RELEASE_VERSION - curl -Ls https://sh.jbang.dev | bash -s - src/test/java/SanityCheck.java - - name: Stop broker - run: docker stop rabbitmq && docker rm rabbitmq - name: Publish Documentation run: | git config user.name "rabbitmq-ci" diff --git a/.github/workflows/sanity-check.yml b/.github/workflows/sanity-check.yml index 26112a1d5f..2baaa423ed 100644 --- a/.github/workflows/sanity-check.yml +++ b/.github/workflows/sanity-check.yml @@ -14,9 +14,9 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout tls-gen - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: rabbitmq/tls-gen path: './tls-gen' diff --git a/.github/workflows/test-native-image.yml b/.github/workflows/test-native-image.yml deleted file mode 100644 index 0b5dcc9dc8..0000000000 --- a/.github/workflows/test-native-image.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Test GraalVM native image - -on: - schedule: - - cron: '0 4 * * *' - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v4 - - name: Checkout tls-gen - uses: actions/checkout@v4 - with: - repository: rabbitmq/tls-gen - path: './tls-gen' - - name: Checkout GraalVM test project - uses: actions/checkout@v4 - with: - repository: rabbitmq/rabbitmq-stream-graal-vm-test - path: './rabbitmq-stream-graal-vm-test' - - name: Set up GraalVM - uses: graalvm/setup-graalvm@v1 - with: - version: 'latest' - java-version: '21' - cache: 'maven' - - name: Start broker - run: ci/start-broker.sh - - name: Install client JAR file - run: | - ./mvnw clean install -Psnapshots -DskipITs -DskipTests -Dgpg.skip=true --no-transfer-progress - export ARTEFACT_VERSION=$(cat pom.xml | grep -oPm1 "(?<=)[^<]+") - echo "artefact_version=$ARTEFACT_VERSION" >> $GITHUB_ENV - - name: Package test application - working-directory: rabbitmq-stream-graal-vm-test - run: | - ./mvnw --version - echo "Using RabbitMQ Stream Java Client ${{ env.artefact_version }}" - ./mvnw clean package -Dstream-client.version=${{ env.artefact_version }} --no-transfer-progress - - name: Use native image program - working-directory: rabbitmq-stream-graal-vm-test - run: target/rabbitmq-stream-graal-vm-test - - name: Stop broker - run: docker stop rabbitmq && docker rm rabbitmq diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 380db0602c..db2a63461b 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout tls-gen - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: rabbitmq/tls-gen path: './tls-gen' diff --git a/.github/workflows/test-rabbitmq-alphas.yml b/.github/workflows/test-rabbitmq-alphas.yml index b707dc290a..79e6040fb4 100644 --- a/.github/workflows/test-rabbitmq-alphas.yml +++ b/.github/workflows/test-rabbitmq-alphas.yml @@ -13,12 +13,14 @@ jobs: runs-on: ubuntu-24.04 strategy: matrix: - rabbitmq-image: [ 'pivotalrabbitmq/rabbitmq:v4.0.x', 'pivotalrabbitmq/rabbitmq:main' ] + rabbitmq-image: + - pivotalrabbitmq/rabbitmq:v4.1.x-otp27 + - pivotalrabbitmq/rabbitmq:main-otp27 name: Test against ${{ matrix.rabbitmq-image }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout tls-gen - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: rabbitmq/tls-gen path: './tls-gen' diff --git a/.github/workflows/test-supported-java-versions.yml b/.github/workflows/test-supported-java-versions.yml index 05834c9d78..51549c2db8 100644 --- a/.github/workflows/test-supported-java-versions.yml +++ b/.github/workflows/test-supported-java-versions.yml @@ -11,15 +11,15 @@ jobs: strategy: matrix: distribution: [ 'temurin' ] - version: [ '11', '17', '21', '23', '24-ea' ] + version: [ '11', '17', '21', '24', '25-ea' ] include: - distribution: 'semeru' version: '17' name: Test against Java ${{ matrix.distribution }} ${{ matrix.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout tls-gen - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: rabbitmq/tls-gen path: './tls-gen' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 523b05cbf9..86381583a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout tls-gen - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: rabbitmq/tls-gen path: './tls-gen' @@ -26,7 +26,7 @@ jobs: distribution: 'temurin' java-version: '21' cache: 'maven' - server-id: ossrh + server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} @@ -60,8 +60,8 @@ jobs: - name: Publish snapshot run: ./mvnw clean deploy -Psnapshots -DskipITs -DskipTests env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} - name: Publish Documentation run: | diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar deleted file mode 100644 index cb28b0e37c..0000000000 Binary files a/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 1a60da7935..12fbe1e907 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4deefa0459..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,96 +0,0 @@ -FROM ubuntu:22.04 as builder - -ARG stream_perf_test_url="set-url-here" - -RUN set -eux; \ - \ - apt-get update; \ - apt-get -y upgrade; \ - apt-get install --yes --no-install-recommends \ - ca-certificates \ - wget \ - gnupg \ - jq - -ARG JAVA_VERSION="21" - -RUN if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then echo "ARM"; ARCH="arm"; BUNDLE="jdk"; else echo "x86"; ARCH="x86"; BUNDLE="jdk"; fi \ - && wget "https://api.azul.com/zulu/download/community/v1.0/bundles/latest/?java_version=$JAVA_VERSION&ext=tar.gz&os=linux&arch=$ARCH&hw_bitness=64&release_status=ga&bundle_type=$BUNDLE" -O jdk-info.json -RUN wget --progress=bar:force:noscroll -O "jdk.tar.gz" $(cat jdk-info.json | jq --raw-output .url) -RUN echo "$(cat jdk-info.json | jq --raw-output .sha256_hash) *jdk.tar.gz" | sha256sum --check --strict - - -RUN set -eux; \ - if [ "$(uname -m)" = "x86_64" ] ; then JAVA_PATH="/usr/lib/jdk-$JAVA_VERSION"; \ - mkdir $JAVA_PATH && \ - tar --extract --file jdk.tar.gz --directory "$JAVA_PATH" --strip-components 1; \ - $JAVA_PATH/bin/jlink --compress=2 --output /jre --add-modules java.base,jdk.management,java.naming,java.xml,jdk.unsupported,jdk.crypto.cryptoki,jdk.httpserver; \ - /jre/bin/java -version; \ - fi - -RUN set -eux; \ - if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ] ; then JAVA_PATH="/jre"; \ - mkdir $JAVA_PATH && \ - tar --extract --file jdk.tar.gz --directory "$JAVA_PATH" --strip-components 1; \ - fi - -# pgpkeys.uk is quite reliable, but allow for substitutions locally -ARG PGP_KEYSERVER=hkps://keys.openpgp.org -# If you are building this image locally and are getting `gpg: keyserver receive failed: No data` errors, -# run the build with a different PGP_KEYSERVER, e.g. docker build --tag rabbitmq:3.7 --build-arg PGP_KEYSERVER=pgpkeys.eu 3.7/ubuntu -# For context, see https://github.com/docker-library/official-images/issues/4252 - -# https://www.rabbitmq.com/signatures.html#importing-gpg -ENV RABBITMQ_PGP_KEY_ID="0x0A9AF2115F4687BD29803A206B73A36E6026DFCA" -ENV STREAM_PERF_TEST_HOME="/stream_perf_test" - -RUN set -eux; \ - \ - wget --progress dot:giga --output-document "/usr/local/src/stream-perf-test.jar.asc" "$stream_perf_test_url.asc"; \ - wget --progress dot:giga --output-document "/usr/local/src/stream-perf-test.jar" "$stream_perf_test_url"; \ - STREAM_PERF_TEST_SHA256="$(wget -qO- $stream_perf_test_url.sha256)"; \ - echo "$STREAM_PERF_TEST_SHA256 /usr/local/src/stream-perf-test.jar" | sha256sum --check --strict -; \ - \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver "$PGP_KEYSERVER" --recv-keys "$RABBITMQ_PGP_KEY_ID"; \ - gpg --batch --verify "/usr/local/src/stream-perf-test.jar.asc" "/usr/local/src/stream-perf-test.jar"; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME"; \ - \ - mkdir -p "$STREAM_PERF_TEST_HOME"; \ - cp /usr/local/src/stream-perf-test.jar $STREAM_PERF_TEST_HOME/stream-perf-test.jar - -FROM ubuntu:22.04 - -# we need locales support for characters like ยต to show up correctly in the console -RUN set -eux; \ - apt-get update; \ - apt-get -y upgrade; \ - apt-get install -y --no-install-recommends \ - locales \ - wget \ - ; \ - rm -rf /var/lib/apt/lists/*; \ - locale-gen en_US.UTF-8 - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk/jre -RUN mkdir -p $JAVA_HOME -COPY --from=builder /jre $JAVA_HOME/ -RUN ln -svT $JAVA_HOME/bin/java /usr/local/bin/java - -RUN mkdir -p /stream_perf_test -WORKDIR /stream_perf_test -COPY --from=builder /stream_perf_test ./ -RUN set -eux; \ - if [ "$(uname -m)" = "x86_64" ] ; then java -jar stream-perf-test.jar --help ; \ - fi - -RUN groupadd --gid 1000 stream-perf-test -RUN useradd --uid 1000 --gid stream-perf-test --comment "perf-test user" stream-perf-test - -USER stream-perf-test:stream-perf-test - -ENTRYPOINT ["java", "-Dio.netty.processId=1", "-jar", "stream-perf-test.jar"] diff --git a/README.adoc b/README.adoc index 95445f6e73..2a96cd7585 100644 --- a/README.adoc +++ b/README.adoc @@ -16,8 +16,7 @@ Please refer to the https://rabbitmq.github.io/rabbitmq-stream-java-client/stabl == Project Maturity -The project is in development and stabilization phase. -Features and API are subject to change, but https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#stability-of-programming-interfaces[breaking changes] will be kept to a minimum. +The library is stable and production-ready. == Support @@ -51,17 +50,7 @@ This library requires at least Java 11, but Java 21 or more is recommended. == Versioning -The RabbitMQ Stream Java Client is in development and stabilization phase. -When the stabilization phase ends, a 1.0.0 version will be cut, and -https://semver.org/[semantic versioning] is likely to be enforced. - -Before reaching the stable phase, the client will use a versioning scheme of `[0.MINOR.PATCH]` where: - -* `0` indicates the project is still in a stabilization phase. -* `MINOR` is a 0-based number incrementing with each new release cycle. It generally reflects significant changes like new features and potentially some programming interfaces changes. -* `PATCH` is a 0-based number incrementing with each service release, that is bux fixes. - -Breaking changes between releases can happen but will be kept to a minimum. +This library uses https://semver.org/[semantic versioning]. == Build Instructions @@ -80,7 +69,7 @@ Launch the broker: ---- docker run -it --rm --name rabbitmq -p 5552:5552 -p 5672:5672 \ -e RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS='-rabbitmq_stream advertised_host localhost' \ - rabbitmq:4.0 + rabbitmq:4.1 ---- Enable the stream plugin: diff --git a/ci/cluster/docker-compose.yml b/ci/cluster/docker-compose.yml index 39345d3e8d..40a6860ee1 100644 --- a/ci/cluster/docker-compose.yml +++ b/ci/cluster/docker-compose.yml @@ -6,7 +6,7 @@ services: - rabbitmq-cluster hostname: node0 container_name: rabbitmq0 - image: ${RABBITMQ_IMAGE:-rabbitmq:4.0} + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} pull_policy: always ports: - "5672:5672" @@ -22,7 +22,7 @@ services: - rabbitmq-cluster hostname: node1 container_name: rabbitmq1 - image: ${RABBITMQ_IMAGE:-rabbitmq:4.0} + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} pull_policy: always ports: - "5673:5672" @@ -38,7 +38,7 @@ services: - rabbitmq-cluster hostname: node2 container_name: rabbitmq2 - image: ${RABBITMQ_IMAGE:-rabbitmq:4.0} + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} pull_policy: always ports: - "5674:5672" diff --git a/ci/evaluate-release.sh b/ci/evaluate-release.sh deleted file mode 100755 index 4ad656d7a0..0000000000 --- a/ci/evaluate-release.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -source ./release-versions.txt - -if [[ $RELEASE_VERSION == *[RCM]* ]] -then - echo "prerelease=true" >> $GITHUB_ENV - echo "ga_release=false" >> $GITHUB_ENV - echo "maven_server_id=packagecloud-rabbitmq-maven-milestones" >> $GITHUB_ENV -else - echo "prerelease=false" >> $GITHUB_ENV - echo "ga_release=true" >> $GITHUB_ENV - echo "maven_server_id=ossrh" >> $GITHUB_ENV -fi \ No newline at end of file diff --git a/ci/start-broker.sh b/ci/start-broker.sh index c13c83ce3f..38c60cc1b9 100755 --- a/ci/start-broker.sh +++ b/ci/start-broker.sh @@ -2,7 +2,7 @@ LOCAL_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0} +RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.1} wait_for_message() { while ! docker logs "$1" | grep -q "$2"; @@ -49,5 +49,6 @@ docker run -d --name rabbitmq \ wait_for_message rabbitmq "completed with" +docker exec rabbitmq rabbitmqctl enable_feature_flag --opt-in khepri_db docker exec rabbitmq rabbitmq-diagnostics erlang_version docker exec rabbitmq rabbitmqctl version diff --git a/ci/start-cluster.sh b/ci/start-cluster.sh index c7a5a9f321..b8440984fb 100755 --- a/ci/start-cluster.sh +++ b/ci/start-cluster.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0} +export RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.1} wait_for_message() { while ! docker logs "$1" | grep -q "$2"; diff --git a/mvnw b/mvnw index 8d937f4c14..19529ddf8c 100755 --- a/mvnw +++ b/mvnw @@ -19,290 +19,241 @@ # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- -# Apache Maven Wrapper startup batch script, version 3.2.0 -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir +# Apache Maven Wrapper startup batch script, version 3.3.2 # # Optional ENV vars # ----------------- -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output # ---------------------------------------------------------------------------- -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /usr/local/etc/mavenrc ] ; then - . /usr/local/etc/mavenrc - fi - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false +# OS specific support. +native_path() { printf %s\\n "$1"; } case "$(uname)" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME - else - JAVA_HOME="/Library/Java/Home"; export JAVA_HOME - fi - fi - ;; +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; esac -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=$(java-config --jre-home) - fi -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --unix "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --unix "$CLASSPATH") -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && - JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="$(which javac)" - if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=$(which readlink) - if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then - if $darwin ; then - javaHome="$(dirname "\"$javaExecutable\"")" - javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" - else - javaExecutable="$(readlink -f "\"$javaExecutable\"")" - fi - javaHome="$(dirname "\"$javaExecutable\"")" - javaHome=$(expr "$javaHome" : '\(.*\)/bin') - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" else JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi fi else - JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi fi +} - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=$(cd "$wdir/.." || exit 1; pwd) - fi - # end of workaround +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" done - printf '%s' "$(cd "$basedir" || exit 1; pwd)" + printf %x\\n $h } -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - # Remove \r in case we run on Windows within Git Bash - # and check out the repository with auto CRLF management - # enabled. Otherwise, we may read lines that are delimited with - # \r\n and produce $'-Xarg\r' rather than -Xarg due to word - # splitting rules. - tr -s '\r\n' ' ' < "$1" - fi +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 } -log() { - if [ "$MVNW_VERBOSE" = true ]; then - printf '%s\n' "$1" - fi +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" } -BASE_DIR=$(find_maven_basedir "$(dirname "$0")") -if [ -z "$BASE_DIR" ]; then - exit 1; +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" fi -MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR -log "$MAVEN_PROJECTBASEDIR" +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" -if [ -r "$wrapperJarPath" ]; then - log "Found $wrapperJarPath" +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT else - log "Couldn't find $wrapperJarPath, downloading it ..." + die "cannot create temp dir" +fi - if [ -n "$MVNW_REPOURL" ]; then - wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - else - wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - fi - while IFS="=" read -r key value; do - # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) - safeValue=$(echo "$value" | tr -d '\r') - case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; - esac - done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" - log "Downloading from: $wrapperUrl" +mkdir -p -- "${MAVEN_HOME%/*}" - if $cygwin; then - wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") - fi +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" - if command -v wget > /dev/null; then - log "Found wget ... using wget" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - else - wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - fi - elif command -v curl > /dev/null; then - log "Found curl ... using curl" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - else - curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - fi - else - log "Falling back to using Java to download" - javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" - javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" - # For Cygwin, switch paths to Windows format before running javac - if $cygwin; then - javaSource=$(cygpath --path --windows "$javaSource") - javaClass=$(cygpath --path --windows "$javaClass") - fi - if [ -e "$javaSource" ]; then - if [ ! -e "$javaClass" ]; then - log " - Compiling MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/javac" "$javaSource") - fi - if [ -e "$javaClass" ]; then - log " - Running MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" - fi - fi - fi +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" fi -########################################################################################## -# End of extension -########################################################################################## -# If specified, validate the SHA-256 sum of the Maven wrapper jar file -wrapperSha256Sum="" -while IFS="=" read -r key value; do - case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; - esac -done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" -if [ -n "$wrapperSha256Sum" ]; then - wrapperSha256Result=false - if command -v sha256sum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then - wrapperSha256Result=true +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true fi - elif command -v shasum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then - wrapperSha256Result=true + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true fi else - echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." - echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 exit 1 fi - if [ $wrapperSha256Result = false ]; then - echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 - echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 - echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 exit 1 fi fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --windows "$CLASSPATH") - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -# shellcheck disable=SC2086 # safe args -exec "$JAVACMD" \ - $MAVEN_OPTS \ - $MAVEN_DEBUG_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd index f80fbad3e7..b150b91ed5 100644 --- a/mvnw.cmd +++ b/mvnw.cmd @@ -1,3 +1,4 @@ +<# : batch portion @REM ---------------------------------------------------------------------------- @REM Licensed to the Apache Software Foundation (ASF) under one @REM or more contributor license agreements. See the NOTICE file @@ -18,188 +19,131 @@ @REM ---------------------------------------------------------------------------- @REM ---------------------------------------------------------------------------- -@REM Apache Maven Wrapper startup batch script, version 3.2.0 -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir +@REM Apache Maven Wrapper startup batch script, version 3.3.2 @REM @REM Optional ENV vars -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output @REM ---------------------------------------------------------------------------- -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* -if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - if "%MVNW_VERBOSE%" == "true" ( - echo Found %WRAPPER_JAR% - ) -) else ( - if not "%MVNW_REPOURL%" == "" ( - SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - ) - if "%MVNW_VERBOSE%" == "true" ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %WRAPPER_URL% - ) - - powershell -Command "&{"^ - "$webclient = new-object System.Net.WebClient;"^ - "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ - "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ - "}"^ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ - "}" - if "%MVNW_VERBOSE%" == "true" ( - echo Finished downloading %WRAPPER_JAR% - ) -) -@REM End of extension - -@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file -SET WRAPPER_SHA_256_SUM="" -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) ) -IF NOT %WRAPPER_SHA_256_SUM%=="" ( - powershell -Command "&{"^ - "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ - "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ - " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ - " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ - " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ - " exit 1;"^ - "}"^ - "}" - if ERRORLEVEL 1 goto error -) - -@REM Provide a "standardized" way to retrieve the CLI args that will -@REM work with both Windows and non-Windows executions. -set MAVEN_CMD_LINE_ARGS=%* - -%MAVEN_JAVA_EXE% ^ - %JVM_CONFIG_MAVEN_PROPS% ^ - %MAVEN_OPTS% ^ - %MAVEN_DEBUG_OPTS% ^ - -classpath %WRAPPER_JAR% ^ - "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ - %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" -if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%"=="on" pause - -if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% - -cmd /C exit /B %ERROR_CODE% +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml index b53b4e1f6c..bf65ae93d7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rabbitmq stream-client - 0.22.0 + 1.3.0-SNAPSHOT RabbitMQ Stream Java Client The RabbitMQ Stream Java client library allows Java applications to interface with @@ -43,64 +43,66 @@ https://github.com/rabbitmq/rabbitmq-stream-java-client scm:git:git://github.com/rabbitmq/rabbitmq-stream-java-client.git scm:git:https://github.com/rabbitmq/rabbitmq-stream-java-client.git - v0.22.0 + HEAD true 1.7.36 1.2.13 - 4.1.117.Final + 4.2.4.Final 0.34.1 - 4.2.30 - 1.14.3 - 13.1.0 + 4.2.33 + 1.15.3 + 13.1.2 4.7.5 - 1.27.1 - 1.5.6-9 + 1.28.0 + 1.5.7-4 1.8.0 - 1.1.10.7 - 5.11.4 - 3.27.3 - 5.15.2 - 5.24.0 - 3.17.0 - 1.17.2 - 2.11.0 - 0.10.5 + 1.1.10.8 + 5.13.4 + 3.27.4 + 5.19.0 + 5.26.0 + 3.18.0 + 1.19.0 + 2.13.1 + 0.10.7 1.2.5 - 1.4.2 + 1.5.3 1.0.4 - 3.13.0 - 3.5.2 + 2.0.72.Final + 3.14.0 + 3.5.3 3.8.1 1.11 - 3.2.7 + 3.2.8 3.2.1 3.3.1 - 3.4.0 + 3.5.0 3.3.1 3.11.2 3.4.2 3.4.0 - 3.1.1 + 3.21.0 + 3.2.0 3.0.0 - 2.3.1 + 3.0.1 + 1.0.3 3.2.1 1.37 - 2.44.2 - 1.25.2 - 0.8.12 - 4.8.6.6 - 4.9.0 + 2.46.1 + 1.28.0 + 0.8.13 + 4.9.3.2 + 4.9.4 - 4.0 + 4.1 6026DFCA yyyy-MM-dd'T'HH:mm:ss'Z' UTF-8 - 0.0.6 - 1.7.0 + 0.8.0 true true @@ -216,6 +218,38 @@ test + + io.netty + netty-transport-native-kqueue + ${netty.version} + osx-aarch_64 + test + + + + io.netty + netty-transport-native-io_uring + ${netty.version} + linux-x86_64 + test + + + + io.netty + netty-tcnative-boringssl-static + ${netty-tcnative.version} + linux-x86_64 + test + + + + io.netty + netty-tcnative-boringssl-static + ${netty-tcnative.version} + osx-aarch_64 + test + + org.assertj assertj-core @@ -422,6 +456,12 @@ + + org.apache.maven.plugins + maven-site-plugin + ${maven-site-plugin.version} + + org.asciidoctor asciidoctor-maven-plugin @@ -437,6 +477,11 @@ asciidoctorj-diagram ${asciidoctorj.diagram.version} + + org.asciidoctor + asciidoctorj-diagram-ditaamini + ${asciidoctorj.diagram.ditaamini.version} + src/docs/asciidoc @@ -613,14 +658,18 @@ + + org.sonatype.central + central-publishing-maven-plugin + ${central-publishing-maven-plugin.version} + true + + central + false + + + - - - io.packagecloud.maven.wagon - maven-packagecloud-wagon - ${maven.packagecloud.wagon.version} - - @@ -631,26 +680,6 @@ false false - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - - - - milestone - - false - false - - - - packagecloud-rabbitmq-maven-milestones - packagecloud+https://packagecloud.io/rabbitmq/maven-milestones - - @@ -659,29 +688,6 @@ false false - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-maven-plugin.version} - true - - ossrh - https://oss.sonatype.org/ - false - 20 - - - - - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - jvm-test-arguments-below-java-21 @@ -702,7 +708,6 @@ - diff --git a/release-versions.txt b/release-versions.txt index 6be9f95466..456b14d5ee 100644 --- a/release-versions.txt +++ b/release-versions.txt @@ -1,5 +1,5 @@ -RELEASE_VERSION="0.22.0" -DEVELOPMENT_VERSION="0.23.0-SNAPSHOT" +RELEASE_VERSION="1.2.0" +DEVELOPMENT_VERSION="1.3.0-SNAPSHOT" RELEASE_BRANCH="main" LATEST=true diff --git a/src/docs/asciidoc/api.adoc b/src/docs/asciidoc/api.adoc index d490a5d12b..2c2d77d14d 100644 --- a/src/docs/asciidoc/api.adoc +++ b/src/docs/asciidoc/api.adoc @@ -49,14 +49,13 @@ The previous snippet uses a URI that specifies the following information: host, username, password, and virtual host (`/`, which is encoded as `%2f`). The URI follows the same rules as the https://www.rabbitmq.com/uri-spec.html[AMQP 0.9.1 URI], -except the protocol must be `rabbitmq-stream`. <> by using -the `rabbitmq-stream+tls` scheme in the URI. +except the protocol must be `rabbitmq-stream`. +<> by using the `rabbitmq-stream+tls` scheme in the URI. When using one URI, the corresponding node will be the main entry point to connect to. The `Environment` will then use the stream protocol to find out more about streams topology -(leaders and replicas) when asked to create `Producer` and `Consumer` instances. The `Environment` -may become blind if this node goes down though, so it may be more appropriate to specify -several other URIs to try in case of failure of a node: +(leaders and replicas) when asked to create `Producer` and `Consumer` instances. +The `Environment` may become blind if this node goes down though, so it may be more appropriate to specify several other URIs to try in case of failure of a node: .Creating an environment with several URIs [source,java,indent=0] @@ -75,7 +74,8 @@ Creating the environment to connect to a cluster node works usually seamlessly. Creating publishers and consumers can cause problems as the client uses hints from the cluster to find the nodes where stream leaders and replicas are located to connect to the appropriate nodes. These connection hints can be accurate or less appropriate depending on the infrastructure. -If you hit some connection problems at some point โ€“ like hostnames impossible to resolve for client applications - this https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams[blog post] should help you understand what is going on and fix the issues. +If you hit connection problems at some point โ€“ like hostnames impossible to resolve for client applications - this https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams[blog post] should help you understand what is going on and fix the issues. +Setting the `advertised_host` and `advertised_port` https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams#advertised-host-and-port[configuration entries] should solve the most common connection problems. To make the local development experience simple, the client library can choose to always use `localhost` for producers and consumers. This happens if the following conditions are met: the initial host to connect to is `localhost`, the user is `guest`, and no custom address resolver has been provided. @@ -88,14 +88,10 @@ TLS can be enabled by using the `rabbitmq-stream+tls` scheme in the URI. The default TLS port is 5551. Use the `EnvironmentBuilder#tls` method to configure TLS. -The most important setting is a `io.netty.handler.ssl.SslContext` instance, -which is created and configured with the -`io.netty.handler.ssl.SslContext#forClient` method. Note hostname verification -is enabled by default. +The most important setting is a `io.netty.handler.ssl.SslContext` instance, which is created and configured with the `io.netty.handler.ssl.SslContext#forClient` method. +Note hostname verification is enabled by default. -The following snippet shows a common configuration, whereby -the client is instructed to trust servers with certificates -signed by the configured certificate authority (CA). +The following snippet shows a common configuration, whereby the client is instructed to trust servers with certificates signed by the configured certificate authority (CA). .Creating an environment that uses TLS [source,java,indent=0] @@ -107,10 +103,14 @@ include::{test-examples}/EnvironmentUsage.java[tag=environment-creation-with-tls <3> Use TLS scheme in environment URI <4> Set `SslContext` in environment configuration -It is sometimes handy to trust any server certificates -in development environments. `EnvironmentBuilder#tls` provides the -`trustEverything` method to do so. **This should -not be used in a production environment**. +Checking the identity of the server the client connects to is an important part of the TLS handshake. +To make this work with the stream client library, it is critical the configured trusted certificates match the hosts returned by cluster nodes in the connection hints. +Make sure to read the section on <>. +You may have to configure the `advertised_host` https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams#advertised-host-and-port[broker setting] in case of a mismatch between trusted certificates and the default connection hints cluster nodes return. + +It is sometimes handy to trust any server certificates in development environments. +`EnvironmentBuilder#tls` provides the `trustEverything` method to do so. +**This should not be used in a production environment**. .Creating a TLS environment that trusts all server certificates for development [source,java,indent=0] @@ -242,15 +242,10 @@ Used as a prefix for connection names. |Configuration helper for TLS. |TLS is enabled if a `rabbitmq-stream+tls` URI is provided. -|`tls#hostnameVerification` -|Enable or disable hostname verification. -|Enabled by default. - |`tls#sslContext` |Set the `io.netty.handler.ssl.SslContext` used for the TLS connection. Use `io.netty.handler.ssl.SslContextBuilder#forClient` to configure it. -The server certificate chain and the client private key are the typical -elements that need to be configured. +The server certificate chain, the client private key, and hostname verification are the usual elements that need to be configured. |The JDK trust manager and no client private key. |`tls#trustEverything` @@ -269,7 +264,7 @@ It is the developer's responsibility to close the `EventLoopGroup` they provide. |`netty#ByteBufAllocator` |`ByteBuf` allocator. -|ByteBufAllocator.DEFAULT +|PooledByteBufAllocator.DEFAULT |`netty#channelCustomizer` |Extension point to customize Netty's `Channel` instances used for connections. diff --git a/src/docs/asciidoc/overview.adoc b/src/docs/asciidoc/overview.adoc index 4e1f736a02..6040e03172 100644 --- a/src/docs/asciidoc/overview.adoc +++ b/src/docs/asciidoc/overview.adoc @@ -75,30 +75,19 @@ recovery and automatic re-subscription for consumers. == Versioning -The RabbitMQ Stream Java Client is in development and stabilization phase. -When the stabilization phase ends, a 1.0.0 version will be cut, and -https://semver.org/[semantic versioning] is likely to be enforced. +This library uses https://semver.org/[semantic versioning]. -Before reaching the stable phase, the client will use a versioning scheme of `[0.MINOR.PATCH]` where: - -* `0` indicates the project is still in a stabilization phase. -* `MINOR` is a 0-based number incrementing with each new release cycle. It generally reflects significant changes like new features and potentially some programming interfaces changes. -* `PATCH` is a 0-based number incrementing with each service release, that is bux fixes. - -Breaking changes between releases can happen but will be kept to a minimum. The next section provides more details about the evolution of programming interfaces. [[stability-of-programming-interfaces]] == Stability of Programming Interfaces -The RabbitMQ Stream Java Client is in active development but its programming interfaces will remain as stable as possible. -There is no guarantee though that they will remain completely stable, at least until it reaches version 1.0.0. - The client contains 2 sets of programming interfaces whose stability are of interest for application developers: * Application Programming Interfaces (API): those are the ones used to write application logic. They include the interfaces and classes in the `com.rabbitmq.stream` package (e.g. `Producer`, `Consumer`, `Message`). -These API constitute the main programming model of the client and will be kept as stable as possible. +These API constitute the main programming model of the client and are kept as stable as possible. +New features may require to add methods to existing interfaces. * Service Provider Interfaces (SPI): those are interfaces to implement mainly technical behavior in the client. They are not meant to be used to implement application logic. Application developers may have to refer to them in the configuration phase and if they want to customize some internal behavior of the client. diff --git a/src/docs/asciidoc/setup.adoc b/src/docs/asciidoc/setup.adoc index e9eaaa6df8..1db3aef926 100644 --- a/src/docs/asciidoc/setup.adoc +++ b/src/docs/asciidoc/setup.adoc @@ -139,8 +139,8 @@ With Maven: - ossrh - https://oss.sonatype.org/content/repositories/snapshots + central-portal-snapshots + https://central.sonatype.com/repository/maven-snapshots/ true false @@ -154,7 +154,14 @@ With Gradle: [source,groovy,subs="attributes,specialcharacters"] ---- repositories { - maven { url 'https://oss.sonatype.org/content/repositories/snapshots' } + maven { + name = 'Central Portal Snapshots' + url = 'https://central.sonatype.com/repository/maven-snapshots/' + // Only search this repository for the specific dependency + content { + includeModule("com.rabbitmq", "{project-artifact-id}") + } + } mavenCentral() } ---- diff --git a/src/main/java/com/rabbitmq/stream/Environment.java b/src/main/java/com/rabbitmq/stream/Environment.java index 00baf4099d..c2a669ea72 100644 --- a/src/main/java/com/rabbitmq/stream/Environment.java +++ b/src/main/java/com/rabbitmq/stream/Environment.java @@ -83,6 +83,22 @@ static EnvironmentBuilder builder() { */ StreamStats queryStreamStats(String stream); + /** + * Store the offset for a given reference on the given stream. + * + *

This method is useful to store a given offset before a consumer is created. + * + *

Prefer the {@link Consumer#store(long)} or {@link MessageHandler.Context#storeOffset()} + * methods to store offsets while consuming messages. + * + * @see Consumer#store(long) + * @see MessageHandler.Context#storeOffset() + * @param reference the reference to store the offset for, e.g. a consumer name + * @param stream the stream + * @param offset the offset to store + */ + void storeOffset(String reference, String stream, long offset); + /** * Return whether a stream exists or not. * diff --git a/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java b/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java index 12dd0643ed..4c8a5239f5 100644 --- a/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java +++ b/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java @@ -436,31 +436,12 @@ EnvironmentBuilder topologyUpdateBackOffDelayPolicy( /** Helper to configure TLS. */ interface TlsConfiguration { - /** - * Enable hostname verification. - * - *

Hostname verification is enabled by default. - * - * @return the TLS configuration helper - */ - TlsConfiguration hostnameVerification(); - - /** - * Enable or disable hostname verification. - * - *

Hostname verification is enabled by default. - * - * @param hostnameVerification - * @return the TLS configuration helper - */ - TlsConfiguration hostnameVerification(boolean hostnameVerification); - /** * Netty {@link SslContext} for TLS connections. * *

Use {@link SslContextBuilder#forClient()} to configure and create an instance. * - * @param sslContext + * @param sslContext the SSL context * @return the TLS configuration helper */ TlsConfiguration sslContext(SslContext sslContext); diff --git a/src/main/java/com/rabbitmq/stream/MessageBuilder.java b/src/main/java/com/rabbitmq/stream/MessageBuilder.java index e91414d90a..02a9d3c3ee 100644 --- a/src/main/java/com/rabbitmq/stream/MessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/MessageBuilder.java @@ -15,6 +15,8 @@ package com.rabbitmq.stream; import java.math.BigDecimal; +import java.util.List; +import java.util.Map; import java.util.UUID; /** @@ -186,6 +188,12 @@ interface MessageAnnotationsBuilder { MessageAnnotationsBuilder entrySymbol(String key, String value); + MessageAnnotationsBuilder entry(String key, List list); + + MessageAnnotationsBuilder entry(String key, Map map); + + MessageAnnotationsBuilder entryArray(String key, Object[] array); + /** * Go back to the message builder * diff --git a/src/main/java/com/rabbitmq/stream/StreamCreator.java b/src/main/java/com/rabbitmq/stream/StreamCreator.java index c37c04fc2d..659a53e728 100644 --- a/src/main/java/com/rabbitmq/stream/StreamCreator.java +++ b/src/main/java/com/rabbitmq/stream/StreamCreator.java @@ -146,23 +146,7 @@ enum LeaderLocator { * *

Default value for RabbitMQ 3.10+. */ - BALANCED("balanced"), - - /** - * The stream leader will be a random node of the cluster. - * - *

Deprecated as of RabbitMQ 3.10, same as {@link LeaderLocator#BALANCED}. - */ - RANDOM("random"), - - /** - * The stream leader will be on the node with the least number of stream leaders. - * - *

Deprecated as of RabbitMQ 3.10, same as {@link LeaderLocator#BALANCED}. - * - *

Default value for RabbitMQ 3.9. - */ - LEAST_LEADERS("least-leaders"); + BALANCED("balanced"); String value; diff --git a/src/main/java/com/rabbitmq/stream/amqp/package-info.java b/src/main/java/com/rabbitmq/stream/amqp/package-info.java index 88cd83e282..14121ad9d6 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/package-info.java +++ b/src/main/java/com/rabbitmq/stream/amqp/package-info.java @@ -1,2 +1,2 @@ -/** Classes for AMQP 1.0 support. */ +/** Classes for AMQP 1.0 message format support. */ package com.rabbitmq.stream.amqp; diff --git a/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java b/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java index 8790128302..abadc2f618 100644 --- a/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java @@ -19,10 +19,7 @@ import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; import java.nio.ByteBuffer; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.function.Function; import org.apache.qpid.proton.amqp.*; import org.apache.qpid.proton.amqp.messaging.*; @@ -63,7 +60,7 @@ private static Map createMapFromAmqpMap( if (amqpMap != null) { result = new LinkedHashMap<>(amqpMap.size()); for (Map.Entry entry : amqpMap.entrySet()) { - result.put(keyMaker.apply(entry.getKey()), convertApplicationProperty(entry.getValue())); + result.put(keyMaker.apply(entry.getKey()), fromQpidToJava(entry.getValue())); } } else { result = null; @@ -71,7 +68,7 @@ private static Map createMapFromAmqpMap( return result; } - private static Object convertApplicationProperty(Object value) { + private static Object fromQpidToJava(Object value) { if (value instanceof Boolean || value instanceof Byte || value instanceof Short @@ -81,7 +78,10 @@ private static Object convertApplicationProperty(Object value) { || value instanceof Double || value instanceof String || value instanceof Character - || value instanceof UUID) { + || value instanceof UUID + || value instanceof List + || value instanceof Map + || value instanceof Object[]) { return value; } else if (value instanceof Binary) { return ((Binary) value).getArray(); @@ -99,9 +99,10 @@ private static Object convertApplicationProperty(Object value) { return ((Symbol) value).toString(); } else if (value == null) { return null; + } else if (value.getClass().isArray()) { + return value; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -281,7 +282,10 @@ protected Object convertToQpidType(Object value) { || value instanceof String || value instanceof Character || value instanceof UUID - || value instanceof Date) { + || value instanceof Date + || value instanceof List + || value instanceof Map + || value instanceof Object[]) { return value; } else if (value instanceof com.rabbitmq.stream.amqp.UnsignedByte) { return UnsignedByte.valueOf(((com.rabbitmq.stream.amqp.UnsignedByte) value).byteValue()); @@ -298,8 +302,7 @@ protected Object convertToQpidType(Object value) { } else if (value == null) { return null; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -634,28 +637,28 @@ public Message copy() { // from // https://github.com/apache/activemq/blob/master/activemq-amqp/src/main/java/org/apache/activemq/transport/amqp/message/AmqpWritableBuffer.java - private static class ByteArrayWritableBuffer implements WritableBuffer { + static class ByteArrayWritableBuffer implements WritableBuffer { - public static final int DEFAULT_CAPACITY = 4 * 1024; + static final int DEFAULT_CAPACITY = 4 * 1024; - byte[] buffer; - int position; + private byte[] buffer; + private int position; /** Creates a new WritableBuffer with default capacity. */ - public ByteArrayWritableBuffer() { + ByteArrayWritableBuffer() { this(DEFAULT_CAPACITY); } /** Create a new WritableBuffer with the given capacity. */ - public ByteArrayWritableBuffer(int capacity) { + ByteArrayWritableBuffer(int capacity) { this.buffer = new byte[capacity]; } - public byte[] getArray() { + byte[] getArray() { return buffer; } - public int getArrayLength() { + int getArrayLength() { return position; } diff --git a/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java index 8902dd1ec1..1d1851bb52 100644 --- a/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java @@ -18,11 +18,9 @@ import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigDecimal; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.qpid.proton.amqp.Binary; import org.apache.qpid.proton.amqp.Symbol; @@ -70,6 +68,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -363,6 +362,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + messageAnnotations.put(Symbol.getSymbol(key), list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + messageAnnotations.put(Symbol.getSymbol(key), map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + messageAnnotations.put(Symbol.getSymbol(key), array); + return this; + } + @Override public MessageBuilder messageBuilder() { return messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java b/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java index a429619585..d7ffeb2c65 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java @@ -18,6 +18,7 @@ import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -107,6 +108,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; diff --git a/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java b/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java index 0feb5d6837..76f15bd806 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java @@ -25,10 +25,9 @@ import com.swiftmq.amqp.v100.types.*; import com.swiftmq.tools.util.DataByteArrayOutputStream; import java.io.IOException; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.lang.reflect.Array; +import java.nio.charset.StandardCharsets; +import java.util.*; public class SwiftMqCodec implements Codec { @@ -69,13 +68,48 @@ private static Object convertAmqpMapValue(AMQPType value) { return ((AMQPUuid) value).getValue(); } else if (value instanceof AMQPSymbol) { return ((AMQPSymbol) value).getValue(); + } else if (value instanceof AMQPList) { + try { + List source = ((AMQPList) value).getValue(); + List target = new ArrayList<>(source.size()); + for (AMQPType o : source) { + target.add(convertAmqpMapValue(o)); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ list", e); + } + } else if (value instanceof AMQPMap) { + try { + Map source = ((AMQPMap) value).getValue(); + Map target = new LinkedHashMap<>(source.size()); + for (Map.Entry entry : source.entrySet()) { + target.put(convertAmqpMapValue(entry.getKey()), convertAmqpMapValue(entry.getValue())); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ map", e); + } + } else if (value instanceof AMQPArray) { + try { + AMQPType[] source = ((AMQPArray) value).getValue(); + Object target = + Array.newInstance( + source.length == 0 ? Object.class : convertAmqpMapValue(source[0]).getClass(), + source.length); + for (int i = 0; i < source.length; i++) { + Array.set(target, i, convertAmqpMapValue(source[i])); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ array", e); + } } else if (value instanceof AMQPNull) { return null; } else if (value == null) { return null; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -320,11 +354,111 @@ protected static AMQPType convertToSwiftMqType(Object value) { return new AMQPSymbol(value.toString()); } else if (value instanceof UUID) { return new AMQPUuid((UUID) value); + } else if (value instanceof List) { + List source = (List) value; + List target = new ArrayList<>(source.size()); + for (Object o : source) { + target.add(convertToSwiftMqType(o)); + } + try { + return new AMQPList(target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ list", e); + } + } else if (value instanceof Map) { + Map source = (Map) value; + Map target = new LinkedHashMap<>(source.size()); + for (Map.Entry entry : source.entrySet()) { + target.put(convertToSwiftMqType(entry.getKey()), convertToSwiftMqType(entry.getValue())); + } + try { + return new AMQPMap(target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ map", e); + } + } else if (value instanceof Object[]) { + Object[] source = (Object[]) value; + AMQPType[] target = new AMQPType[source.length]; + for (int i = 0; i < source.length; i++) { + target[i] = convertToSwiftMqType(source[i]); + } + try { + int code = source.length == 0 ? AMQPTypeDecoder.UNKNOWN : toSwiftMqTypeCode(source[0]); + return new AMQPArray(code, target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ list", e); + } } else if (value == null) { return AMQPNull.NULL; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); + } + } + + protected static int toSwiftMqTypeCode(Object value) { + if (value instanceof Boolean) { + return AMQPTypeDecoder.BOOLEAN; + } else if (value instanceof Byte) { + return AMQPTypeDecoder.BYTE; + } else if (value instanceof Short) { + return AMQPTypeDecoder.SHORT; + } else if (value instanceof Integer) { + int v = (Integer) value; + return (v < -128 || v > 127) ? AMQPTypeDecoder.INT : AMQPTypeDecoder.SINT; + } else if (value instanceof Long) { + long v = (Long) value; + return (v < -128 || v > 127) ? AMQPTypeDecoder.LONG : AMQPTypeDecoder.SLONG; + } else if (value instanceof UnsignedByte) { + return AMQPTypeDecoder.UBYTE; + } else if (value instanceof UnsignedShort) { + return AMQPTypeDecoder.USHORT; + } else if (value instanceof UnsignedInteger) { + return AMQPTypeDecoder.UINT; + } else if (value instanceof UnsignedLong) { + return AMQPTypeDecoder.ULONG; + } else if (value instanceof Float) { + return AMQPTypeDecoder.FLOAT; + } else if (value instanceof Double) { + return AMQPTypeDecoder.DOUBLE; + } else if (value instanceof byte[]) { + return ((byte[]) value).length > 255 ? AMQPTypeDecoder.BIN32 : AMQPTypeDecoder.BIN8; + } else if (value instanceof String) { + return value.toString().getBytes(StandardCharsets.UTF_8).length > 255 + ? AMQPTypeDecoder.STR32UTF8 + : AMQPTypeDecoder.STR8UTF8; + } else if (value instanceof Character) { + return AMQPTypeDecoder.CHAR; + } else if (value instanceof Date) { + return AMQPTypeDecoder.TIMESTAMP; + } else if (value instanceof Symbol) { + return value.toString().getBytes(StandardCharsets.US_ASCII).length > 255 + ? AMQPTypeDecoder.SYM32 + : AMQPTypeDecoder.SYM8; + } else if (value instanceof UUID) { + return AMQPTypeDecoder.UUID; + } else if (value instanceof List) { + List l = (List) value; + if (l.isEmpty()) { + return AMQPTypeDecoder.LIST0; + } else if (l.size() > 255) { + return AMQPTypeDecoder.LIST32; + } else { + return AMQPTypeDecoder.LIST8; + } + } else if (value instanceof Map) { + Map source = (Map) value; + return source.size() * 2 > 255 ? AMQPTypeDecoder.MAP32 : AMQPTypeDecoder.MAP8; + } else if (value instanceof Object[]) { + Object[] source = (Object[]) value; + if (source.length > 255) { + return AMQPTypeDecoder.ARRAY32; + } else { + return AMQPTypeDecoder.ARRAY8; + } + } else if (value == null) { + return AMQPTypeDecoder.NULL; + } else { + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } diff --git a/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java index a6180c9655..9e4e8ea8ee 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java @@ -14,6 +14,9 @@ // info@rabbitmq.com. package com.rabbitmq.stream.codec; +import static com.rabbitmq.stream.codec.SwiftMqCodec.convertToSwiftMqType; +import static com.rabbitmq.stream.codec.SwiftMqCodec.toSwiftMqTypeCode; + import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.StreamException; @@ -21,11 +24,11 @@ import com.swiftmq.amqp.v100.generated.transport.definitions.SequenceNo; import com.swiftmq.amqp.v100.messaging.AMQPMessage; import com.swiftmq.amqp.v100.types.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; +import java.lang.reflect.Array; import java.math.BigDecimal; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -77,6 +80,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -319,6 +323,62 @@ protected void addEntry(String key, String value) { protected void addEntrySymbol(String key, String value) { map.put(keyMaker.apply(key), value == null ? AMQPNull.NULL : new AMQPSymbol(value)); } + + protected void addEntry(String key, List list) { + AMQPType amqpValue; + if (list == null) { + amqpValue = AMQPNull.NULL; + } else { + List l = new ArrayList<>(list.size()); + for (Object o : list) { + l.add(convertToSwiftMqType(o)); + } + try { + amqpValue = new AMQPList(l); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMq list", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } + + protected void addEntry(String key, Map mapEntry) { + AMQPType amqpValue; + if (mapEntry == null) { + amqpValue = AMQPNull.NULL; + } else { + Map m = new LinkedHashMap<>(mapEntry.size()); + mapEntry.forEach( + (k, v) -> { + m.put(convertToSwiftMqType(k), convertToSwiftMqType(v)); + }); + try { + amqpValue = new AMQPMap(m); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ map", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } + + protected void addEntry(String key, Object[] array) { + AMQPType amqpValue; + if (array == null) { + amqpValue = AMQPNull.NULL; + } else { + AMQPType[] a = new AMQPType[array.length]; + for (int i = 0; i < array.length; i++) { + a[i] = convertToSwiftMqType(Array.get(array, i)); + } + try { + int code = a.length == 0 ? AMQPTypeDecoder.UNKNOWN : toSwiftMqTypeCode(array[0]); + amqpValue = new AMQPArray(code, a); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMq list", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } } private static class SwiftMqApplicationPropertiesBuilder extends AmqpMapBuilderSupport @@ -585,6 +645,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + addEntry(key, list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + addEntry(key, map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + addEntry(key, array); + return this; + } + @Override public MessageBuilder messageBuilder() { return messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java index 195d37cb89..e1c1afc850 100644 --- a/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java @@ -18,11 +18,9 @@ import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; import com.rabbitmq.stream.amqp.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigDecimal; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; public class WrapperMessageBuilder implements MessageBuilder { @@ -56,6 +54,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -218,6 +217,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + messageAnnotations.put(key, list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + messageAnnotations.put(key, map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + messageAnnotations.put(key, array); + return this; + } + @Override public MessageBuilder messageBuilder() { return this.messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/impl/Client.java b/src/main/java/com/rabbitmq/stream/impl/Client.java index 9e800163a1..df5abb0f6b 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Client.java +++ b/src/main/java/com/rabbitmq/stream/impl/Client.java @@ -59,17 +59,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufOutputStream; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelOutboundHandlerAdapter; -import io.netty.channel.ChannelPromise; -import io.netty.channel.ConnectTimeoutException; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.*; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.DecoderException; @@ -106,9 +96,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.function.ToLongFunction; -import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -129,6 +117,7 @@ public class Client implements AutoCloseable { public static final int DEFAULT_PORT = 5552; public static final int DEFAULT_TLS_PORT = 5551; static final int MAX_REFERENCE_SIZE = 256; + static final int DEFAULT_MAX_FRAME_SIZE = 1048576; static final OutboundEntityWriteCallback OUTBOUND_MESSAGE_WRITE_CALLBACK = new OutboundMessageWriteCallback(); static final OutboundEntityWriteCallback OUTBOUND_MESSAGE_BATCH_WRITE_CALLBACK = @@ -155,8 +144,12 @@ public class Client implements AutoCloseable { final ConcurrentMap> outstandingRequests = new ConcurrentHashMap<>(); final List subscriptionOffsets = new CopyOnWriteArrayList<>(); + // dispatches broker frames, except for delivery frames final ExecutorService executorService; + private final Consumer closeExecutorService; + // dispatches delivery frames only final ExecutorService dispatchingExecutorService; + private final Consumer closeDispatchingExecutorService; final TuneState tuneState; final AtomicBoolean closing = new AtomicBoolean(false); final AtomicBoolean shuttingDownDispatching = new AtomicBoolean(false); @@ -174,7 +167,6 @@ public long applyAsLong(Object value) { } }; private final AtomicInteger correlationSequence = new AtomicInteger(0); - private final Runnable executorServiceClosing; private final SaslConfiguration saslConfiguration; private final CredentialsProvider credentialsProvider; private final Runnable nettyClosing; @@ -245,7 +237,7 @@ public Client(ClientParameters parameters) { if (b.config().group() == null) { EventLoopGroup eventLoopGroup; if (parameters.eventLoopGroup == null) { - this.eventLoopGroup = new NioEventLoopGroup(); + this.eventLoopGroup = Utils.eventLoopGroup(); eventLoopGroup = this.eventLoopGroup; } else { this.eventLoopGroup = null; @@ -265,7 +257,7 @@ public Client(ClientParameters parameters) { b.option( ChannelOption.ALLOCATOR, parameters.byteBufAllocator == null - ? ByteBufAllocator.DEFAULT + ? Utils.byteBufAllocator() : parameters.byteBufAllocator); } @@ -290,13 +282,6 @@ public void initChannel(SocketChannel ch) { SslHandler sslHandler = parameters.sslContext.newHandler(ch.alloc(), parameters.host, parameters.port); - if (parameters.tlsHostnameVerification) { - SSLEngine sslEngine = sslHandler.engine(); - SSLParameters sslParameters = sslEngine.getSSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - sslEngine.setSSLParameters(sslParameters); - } - ch.pipeline().addFirst("ssl", sslHandler); } channelCustomizer.accept(ch); @@ -331,44 +316,58 @@ public void initChannel(SocketChannel ch) { this.channel = f.channel(); ExecutorServiceFactory executorServiceFactory = parameters.executorServiceFactory; if (executorServiceFactory == null) { + this.closeExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + es.shutdownNow(); + } + }); this.executorService = Executors.newSingleThreadExecutor(threadFactory(clientConnectionName + "-")); } else { + this.closeExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + executorServiceFactory.clientClosed(es); + } + }); this.executorService = executorServiceFactory.get(); } ExecutorServiceFactory dispatchingExecutorServiceFactory = parameters.dispatchingExecutorServiceFactory; if (dispatchingExecutorServiceFactory == null) { + this.closeDispatchingExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + List outstandingTasks = es.shutdownNow(); + this.shuttingDownDispatching.set(true); + for (Runnable outstandingTask : outstandingTasks) { + try { + outstandingTask.run(); + } catch (Exception e) { + LOGGER.info( + "Error while releasing buffer in outstanding connection tasks: {}", + e.getMessage()); + } + } + } + }); this.dispatchingExecutorService = Executors.newSingleThreadExecutor( - threadFactory("dispatching-" + clientConnectionName + "-")); + threadFactory("rabbitmq-stream-dispatching-" + clientConnectionName + "-")); } else { + this.closeDispatchingExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + dispatchingExecutorServiceFactory.clientClosed(es); + } + }); this.dispatchingExecutorService = dispatchingExecutorServiceFactory.get(); } - this.executorServiceClosing = - Utils.makeIdempotent( - () -> { - if (dispatchingExecutorServiceFactory == null) { - List outstandingTasks = this.dispatchingExecutorService.shutdownNow(); - this.shuttingDownDispatching.set(true); - for (Runnable outstandingTask : outstandingTasks) { - try { - outstandingTask.run(); - } catch (Exception e) { - LOGGER.info( - "Error while releasing buffer in outstanding connection tasks: {}", - e.getMessage()); - } - } - } else { - dispatchingExecutorServiceFactory.clientClosed(this.dispatchingExecutorService); - } - if (executorServiceFactory == null) { - this.executorService.shutdownNow(); - } else { - executorServiceFactory.clientClosed(this.executorService); - } - }); try { this.tuneState = new TuneState( @@ -1451,7 +1450,12 @@ void closingSequence(ShutdownContext.ShutdownReason reason) { this.shutdownListenerCallback.accept(reason); } this.nettyClosing.run(); - this.executorServiceClosing.run(); + if (this.closeDispatchingExecutorService != null) { + this.closeDispatchingExecutorService.accept(this.dispatchingExecutorService); + } + if (this.closeExecutorService != null) { + this.closeExecutorService.accept(this.executorService); + } } private void closeNetty() { @@ -2360,7 +2364,7 @@ public static class ClientParameters { CompressionCodecFactory compressionCodecFactory; private String virtualHost = "/"; private Duration requestedHeartbeat = Duration.ofSeconds(60); - private int requestedMaxFrameSize = 1048576; + private int requestedMaxFrameSize = DEFAULT_MAX_FRAME_SIZE; private PublishConfirmListener publishConfirmListener = NO_OP_PUBLISH_CONFIRM_LISTENER; private PublishErrorListener publishErrorListener = NO_OP_PUBLISH_ERROR_LISTENER; private ChunkListener chunkListener = @@ -2385,7 +2389,6 @@ public static class ClientParameters { private ChunkChecksum chunkChecksum = JdkChunkChecksum.CRC32_SINGLETON; private MetricsCollector metricsCollector = NoOpMetricsCollector.SINGLETON; private SslContext sslContext; - private boolean tlsHostnameVerification = true; private ByteBufAllocator byteBufAllocator; private Duration rpcTimeout; private Consumer channelCustomizer = noOpConsumer(); @@ -2542,11 +2545,6 @@ public ClientParameters sslContext(SslContext sslContext) { return this; } - public ClientParameters tlsHostnameVerification(boolean tlsHostnameVerification) { - this.tlsHostnameVerification = tlsHostnameVerification; - return this; - } - public ClientParameters compressionCodecFactory( CompressionCodecFactory compressionCodecFactory) { this.compressionCodecFactory = compressionCodecFactory; diff --git a/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java b/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java deleted file mode 100644 index f83d2abf29..0000000000 --- a/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. -// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. -// -// This software, the RabbitMQ Stream Java client library, is dual-licensed under the -// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). -// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, -// please see LICENSE-APACHE2. -// -// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, -// either express or implied. See the LICENSE file for specific language governing -// rights and limitations of this software. -// -// If you have any questions regarding licensing, please contact us at -// info@rabbitmq.com. -package com.rabbitmq.stream.impl; - -import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -final class ConcurrencyUtils { - - private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrencyUtils.class); - - private static final ThreadFactory THREAD_FACTORY; - - static { - if (isJava21OrMore()) { - LOGGER.debug("Running Java 21 or more, using virtual threads"); - Class builderClass = - Arrays.stream(Thread.class.getDeclaredClasses()) - .filter(c -> "Builder".equals(c.getSimpleName())) - .findFirst() - .get(); - // Reflection code is the same as: - // Thread.ofVirtual().factory(); - try { - Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); - THREAD_FACTORY = (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - throw new RuntimeException(e); - } - } else { - THREAD_FACTORY = Executors.defaultThreadFactory(); - } - } - - private ConcurrencyUtils() {} - - static ThreadFactory defaultThreadFactory() { - return THREAD_FACTORY; - } - - private static boolean isJava21OrMore() { - String version = System.getProperty("java.version").replace("-beta", ""); - return Utils.versionCompare(version, "21.0") >= 0; - } -} diff --git a/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java b/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java index 3faddae938..05a9ae00c1 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java +++ b/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java @@ -75,7 +75,7 @@ final class ConsumersCoordinator implements AutoCloseable { private final List trackers = new CopyOnWriteArrayList<>(); private final ExecutorServiceFactory executorServiceFactory = new DefaultExecutorServiceFactory( - Runtime.getRuntime().availableProcessors(), 10, "rabbitmq-stream-consumer-connection-"); + AVAILABLE_PROCESSORS, 10, "rabbitmq-stream-consumer-connection-"); private final boolean forceReplica; private final Lock coordinatorLock = new ReentrantLock(); diff --git a/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java b/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java index 7e2d1a7369..c4bc94d639 100644 --- a/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java +++ b/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java @@ -28,18 +28,23 @@ final class DynamicBatch implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(DynamicBatch.class); - private static final int MIN_BATCH_SIZE = 32; - private static final int MAX_BATCH_SIZE = 8192; + private static final int MIN_BATCH_SIZE = 16; private final BlockingQueue requests = new LinkedBlockingQueue<>(); private final BatchConsumer consumer; - private final int configuredBatchSize; + private final int configuredBatchSize, minBatchSize, maxBatchSize; private final Thread thread; - DynamicBatch(BatchConsumer consumer, int batchSize) { + DynamicBatch(BatchConsumer consumer, int batchSize, int maxUnconfirmed, String id) { this.consumer = consumer; - this.configuredBatchSize = min(max(batchSize, MIN_BATCH_SIZE), MAX_BATCH_SIZE); - this.thread = ConcurrencyUtils.defaultThreadFactory().newThread(this::loop); + if (batchSize < maxUnconfirmed) { + this.minBatchSize = min(MIN_BATCH_SIZE, batchSize / 2); + } else { + this.minBatchSize = min(1, maxUnconfirmed / 2); + } + this.configuredBatchSize = batchSize; + this.maxBatchSize = batchSize * 2; + this.thread = ThreadUtils.newInternalThread(id, this::loop); this.thread.start(); } @@ -69,15 +74,7 @@ private void loop() { if (state.items.size() >= state.batchSize) { this.maybeCompleteBatch(state, true); } else { - item = this.requests.poll(); - if (item == null) { - this.maybeCompleteBatch(state, false); - } else { - state.items.add(item); - if (state.items.size() >= state.batchSize) { - this.maybeCompleteBatch(state, true); - } - } + pump(state, 2); } } else { this.maybeCompleteBatch(state, false); @@ -85,6 +82,22 @@ private void loop() { } } + private void pump(State state, int pumpCount) { + if (pumpCount <= 0) { + return; + } + T item = this.requests.poll(); + if (item == null) { + this.maybeCompleteBatch(state, false); + } else { + state.items.add(item); + if (state.items.size() >= state.batchSize) { + this.maybeCompleteBatch(state, true); + } + this.pump(state, pumpCount - 1); + } + } + private static final class State { int batchSize; @@ -96,13 +109,14 @@ private void maybeCompleteBatch(State state, boolean increaseIfCompleted) { boolean completed = this.consumer.process(state.items); if (completed) { if (increaseIfCompleted) { - state.batchSize = min(state.batchSize * 2, MAX_BATCH_SIZE); + state.batchSize = min(state.batchSize * 2, this.maxBatchSize); } else { - state.batchSize = max(state.batchSize / 2, MIN_BATCH_SIZE); + state.batchSize = max(state.batchSize / 2, this.minBatchSize); } state.items = new ArrayList<>(state.batchSize); } } catch (Exception e) { + // e.printStackTrace(); LOGGER.warn("Error during dynamic batch completion: {}", e.getMessage()); } } diff --git a/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java b/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java index ee8c397e13..b4020dc3ce 100644 --- a/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java +++ b/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java @@ -38,6 +38,7 @@ final class DynamicBatchMessageAccumulator implements MessageAccumulator { DynamicBatchMessageAccumulator( int subEntrySize, int batchSize, + int maxUnconfirmedMessages, Codec codec, int maxFrameSize, ToLongFunction publishSequenceFunction, @@ -47,7 +48,8 @@ final class DynamicBatchMessageAccumulator implements MessageAccumulator { CompressionCodec compressionCodec, ByteBufAllocator byteBufAllocator, ObservationCollector observationCollector, - StreamProducer producer) { + StreamProducer producer, + long producerId) { this.helper = new ProducerUtils.MessageAccumulatorHelper( codec, @@ -60,6 +62,7 @@ final class DynamicBatchMessageAccumulator implements MessageAccumulator { this.producer = producer; this.observationCollector = (ObservationCollector) observationCollector; boolean shouldObserve = !this.observationCollector.isNoop(); + String batchId = "rabbitmq-stream-dynamic-batch-producer-" + producerId; if (subEntrySize <= 1) { this.dynamicBatch = new DynamicBatch<>( @@ -75,7 +78,9 @@ final class DynamicBatchMessageAccumulator implements MessageAccumulator { } return result; }, - batchSize); + batchSize, + maxUnconfirmedMessages, + batchId); } else { byte compressionCode = compressionCodec == null ? Compression.NONE.code() : compressionCodec.code(); @@ -124,7 +129,9 @@ final class DynamicBatchMessageAccumulator implements MessageAccumulator { } return result; }, - batchSize * subEntrySize); + batchSize * subEntrySize, + maxUnconfirmedMessages, + batchId); } } diff --git a/src/main/java/com/rabbitmq/stream/impl/HashUtils.java b/src/main/java/com/rabbitmq/stream/impl/HashUtils.java index f4f235aff4..fde6d946b2 100644 --- a/src/main/java/com/rabbitmq/stream/impl/HashUtils.java +++ b/src/main/java/com/rabbitmq/stream/impl/HashUtils.java @@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.util.function.ToIntFunction; +@SuppressFBWarnings({"SF_SWITCH_FALLTHROUGH", "SF_SWITCH_NO_DEFAULT"}) final class HashUtils { static final ToIntFunction MURMUR3 = new Murmur3(); @@ -72,7 +73,6 @@ private static int fmix32(int hash) { this.seed = seed; } - @SuppressFBWarnings({"SF_SWITCH_FALLTHROUGH", "SF_SWITCH_NO_DEFAULT"}) @Override public int applyAsInt(String value) { byte[] data = value.getBytes(StandardCharsets.UTF_8); diff --git a/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingUtils.java b/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingUtils.java new file mode 100644 index 0000000000..74aad57628 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingUtils.java @@ -0,0 +1,125 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static com.rabbitmq.stream.BackOffDelayPolicy.fixedWithInitialDelay; +import static com.rabbitmq.stream.impl.AsyncRetry.asyncRetry; +import static java.lang.String.format; +import static java.time.Duration.ofMillis; + +import com.rabbitmq.stream.Constants; +import com.rabbitmq.stream.NoOffsetException; +import com.rabbitmq.stream.StreamException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class OffsetTrackingUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(OffsetTrackingUtils.class); + + private OffsetTrackingUtils() {} + + static long storedOffset(Supplier clientSupplier, String name, String stream) { + // the client can be null, so we catch any exception + Client.QueryOffsetResponse response; + try { + response = clientSupplier.get().queryOffset(name, stream); + } catch (Exception e) { + throw new IllegalStateException( + format( + "Not possible to query offset for name %s on stream %s for now: %s", + name, stream, e.getMessage()), + e); + } + if (response.isOk()) { + return response.getOffset(); + } else if (response.getResponseCode() == Constants.RESPONSE_CODE_NO_OFFSET) { + throw new NoOffsetException( + format( + "No offset stored for name %s on stream %s (%s)", + name, stream, Utils.formatConstant(response.getResponseCode()))); + } else { + throw new StreamException( + format( + "QueryOffset for name %s on stream %s returned an error (%s)", + name, stream, Utils.formatConstant(response.getResponseCode())), + response.getResponseCode()); + } + } + + static void waitForOffsetToBeStored( + String caller, + ScheduledExecutorService scheduledExecutorService, + LongSupplier offsetSupplier, + String name, + String stream, + long expectedStoredOffset) { + String reference = String.format("{stream=%s/name=%s}", stream, name); + CompletableFuture storedTask = + asyncRetry( + () -> { + try { + long lastStoredOffset = offsetSupplier.getAsLong(); + boolean stored = lastStoredOffset == expectedStoredOffset; + LOGGER.debug( + "Last stored offset from {} on {} is {}, expecting {}", + caller, + reference, + lastStoredOffset, + expectedStoredOffset); + if (!stored) { + throw new IllegalStateException(); + } else { + return true; + } + } catch (StreamException e) { + if (e.getCode() == Constants.RESPONSE_CODE_NO_OFFSET) { + LOGGER.debug( + "No stored offset for {} on {}, expecting {}", + caller, + reference, + expectedStoredOffset); + throw new IllegalStateException(); + } else { + throw e; + } + } + }) + .description( + "Last stored offset for %s on %s must be %d", + caller, reference, expectedStoredOffset) + .delayPolicy(fixedWithInitialDelay(ofMillis(200), ofMillis(200))) + .retry(exception -> exception instanceof IllegalStateException) + .scheduler(scheduledExecutorService) + .build(); + + try { + storedTask.get(10, TimeUnit.SECONDS); + LOGGER.debug("Offset {} stored ({}, {})", expectedStoredOffset, caller, reference); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException | TimeoutException e) { + LOGGER.warn("Error while checking offset has been stored", e); + storedTask.cancel(true); + } + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java b/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java index 5ae8faa7dd..225d450087 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java +++ b/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java @@ -30,6 +30,7 @@ static MessageAccumulator createMessageAccumulator( boolean dynamicBatch, int subEntrySize, int batchSize, + int maxUnconfirmedMessages, CompressionCodec compressionCodec, Codec codec, ByteBufAllocator byteBufAllocator, @@ -39,11 +40,13 @@ static MessageAccumulator createMessageAccumulator( Clock clock, String stream, ObservationCollector observationCollector, - StreamProducer producer) { + StreamProducer producer, + long producerId) { if (dynamicBatch) { return new DynamicBatchMessageAccumulator( subEntrySize, batchSize, + maxUnconfirmedMessages, codec, maxFrameSize, publishSequenceFunction, @@ -53,7 +56,8 @@ static MessageAccumulator createMessageAccumulator( compressionCodec, byteBufAllocator, observationCollector, - producer); + producer, + producerId); } else { if (subEntrySize <= 1) { return new SimpleMessageAccumulator( diff --git a/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java b/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java index 67a874df9b..8ac1017f8a 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java +++ b/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java @@ -68,7 +68,7 @@ final class ProducersCoordinator implements AutoCloseable { private final List producerTrackers = new CopyOnWriteArrayList<>(); private final ExecutorServiceFactory executorServiceFactory = new DefaultExecutorServiceFactory( - Runtime.getRuntime().availableProcessors(), 10, "rabbitmq-stream-producer-connection-"); + AVAILABLE_PROCESSORS, 10, "rabbitmq-stream-producer-connection-"); private final Lock coordinatorLock = new ReentrantLock(); private final boolean forceLeader; @@ -750,7 +750,10 @@ private void assignProducersToNewManagers( List candidates = brokerAndCandidates.v2(); String key = keyForNode(broker); LOGGER.debug( - "Assigning {} producer(s) and consumer tracker(s) to {} (stream '{}')", trackers.size(), key, stream); + "Assigning {} producer(s) and consumer tracker(s) to {} (stream '{}')", + trackers.size(), + key, + stream); trackers.forEach(tracker -> maybeRecoverAgent(broker, candidates, tracker)); }) .exceptionally( diff --git a/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java b/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java index 76eaded5e8..049d9a4326 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java +++ b/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java @@ -128,7 +128,7 @@ public void shutdown() { @Override public List shutdownNow() { - this.delegate.shutdownNow(); + this.scheduler.shutdownNow(); return this.delegate.shutdownNow(); } diff --git a/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java b/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java index c8410bbd0f..d63da3ab30 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java +++ b/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java @@ -332,9 +332,14 @@ static int handleMessage( if (ignore && Long.compareUnsigned(offset, offsetLimit) < 0) { messageIgnored.set(true); } else { - Message message = codec.decode(data); - messageListener.handle( - subscriptionId, offset, chunkTimestamp, committedChunkId, chunkContext, message); + try { + Message message = codec.decode(data); + messageListener.handle( + subscriptionId, offset, chunkTimestamp, committedChunkId, chunkContext, message); + } catch (RuntimeException e) { + LOGGER.warn("Error while decoding message at offset {}", offset, e); + throw e; + } } return read; } @@ -451,7 +456,6 @@ static int handleDeliver( } metricsCollector.chunk(numEntries); - long messagesRead = 0; MutableBoolean messageIgnored = new MutableBoolean(false); while (numRecords != 0) { @@ -482,7 +486,7 @@ static int handleDeliver( subscriptionId, offset, chunkTimestamp, committedOffset, chunkContext); messageIgnored.set(false); } else { - messagesRead++; + metricsCollector.consume(1); } numRecords--; offset++; // works even for unsigned long @@ -551,7 +555,7 @@ static int handleDeliver( subscriptionId, offset, chunkTimestamp, committedOffset, chunkContext); messageIgnored.set(false); } else { - messagesRead++; + metricsCollector.consume(1); } numRecordsInBatch--; offset++; // works even for unsigned long @@ -564,7 +568,6 @@ static int handleDeliver( } } } - metricsCollector.consume(messagesRead); return read; } diff --git a/src/main/java/com/rabbitmq/stream/impl/ShutdownService.java b/src/main/java/com/rabbitmq/stream/impl/ShutdownService.java new file mode 100644 index 0000000000..ef59dc5823 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/ShutdownService.java @@ -0,0 +1,85 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper to register callbacks and call them in reverse order. Registered callbacks are made + * automatically idempotent. + * + *

This class can be used to register closing callbacks, call them individually, and/or call all + * of them (in LIFO order) with the {@link #close()} method. + * + *

From + * https://github.com/rabbitmq/rabbitmq-perf-test/blob/main/src/main/java/com/rabbitmq/perf/ShutdownService.java. + */ +final class ShutdownService implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ShutdownService.class); + + private final List closeables = Collections.synchronizedList(new ArrayList<>()); + + /** + * Wrap and register the callback into an idempotent {@link AutoCloseable}. + * + * @param closeCallback + * @return the callback as an idempotent {@link AutoCloseable} + */ + AutoCloseable wrap(CloseCallback closeCallback) { + AtomicBoolean closingOrAlreadyClosed = new AtomicBoolean(false); + AutoCloseable idempotentCloseCallback = + new AutoCloseable() { + @Override + public void close() throws Exception { + if (closingOrAlreadyClosed.compareAndSet(false, true)) { + closeCallback.run(); + } + } + + @Override + public String toString() { + return closeCallback.toString(); + } + }; + closeables.add(idempotentCloseCallback); + return idempotentCloseCallback; + } + + /** Close all the registered callbacks, in the reverse order of registration. */ + @Override + public synchronized void close() { + if (!closeables.isEmpty()) { + for (int i = closeables.size() - 1; i >= 0; i--) { + try { + closeables.get(i).close(); + } catch (Exception e) { + LOGGER.warn("Could not properly execute closing step '{}'", closeables.get(i), e); + } + } + } + } + + @FunctionalInterface + interface CloseCallback { + + void run() throws Exception; + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java b/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java index 591b6db2e2..11c57e4517 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java @@ -18,11 +18,9 @@ import static com.rabbitmq.stream.impl.AsyncRetry.asyncRetry; import static com.rabbitmq.stream.impl.Utils.offsetBefore; import static java.lang.String.format; -import static java.time.Duration.ofMillis; import com.rabbitmq.stream.*; import com.rabbitmq.stream.MessageHandler.Context; -import com.rabbitmq.stream.impl.Client.QueryOffsetResponse; import com.rabbitmq.stream.impl.StreamConsumerBuilder.TrackingConfiguration; import com.rabbitmq.stream.impl.StreamEnvironment.LocatorNotAvailableException; import com.rabbitmq.stream.impl.StreamEnvironment.TrackingConsumerRegistration; @@ -329,53 +327,13 @@ static long getStoredOffsetSafely(StreamConsumer consumer, StreamEnvironment env } void waitForOffsetToBeStored(long expectedStoredOffset) { - CompletableFuture storedTask = - asyncRetry( - () -> { - try { - long lastStoredOffset = storedOffset(); - boolean stored = lastStoredOffset == expectedStoredOffset; - LOGGER.debug( - "Last stored offset from consumer {} on {} is {}, expecting {}", - this.id, - this.stream, - lastStoredOffset, - expectedStoredOffset); - if (!stored) { - throw new IllegalStateException(); - } else { - return true; - } - } catch (StreamException e) { - if (e.getCode() == Constants.RESPONSE_CODE_NO_OFFSET) { - LOGGER.debug( - "No stored offset for consumer {} on {}, expecting {}", - this.id, - this.stream, - expectedStoredOffset); - throw new IllegalStateException(); - } else { - throw e; - } - } - }) - .description( - "Last stored offset for consumer %s on stream %s must be %d", - this.name, this.stream, expectedStoredOffset) - .delayPolicy(fixedWithInitialDelay(ofMillis(200), ofMillis(200))) - .retry(exception -> exception instanceof IllegalStateException) - .scheduler(environment.scheduledExecutorService()) - .build(); - - try { - storedTask.get(10, TimeUnit.SECONDS); - LOGGER.debug( - "Offset {} stored (consumer {}, stream {})", expectedStoredOffset, this.id, this.stream); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (ExecutionException | TimeoutException e) { - LOGGER.warn("Error while checking offset has been stored", e); - } + OffsetTrackingUtils.waitForOffsetToBeStored( + "consumer " + this.id, + this.environment.scheduledExecutorService(), + this::storedOffset, + this.name, + this.stream, + expectedStoredOffset); } void start() { @@ -563,31 +521,7 @@ void running() { long storedOffset(Supplier clientSupplier) { checkNotClosed(); if (canTrack()) { - // the client can be null by now, so we catch any exception - QueryOffsetResponse response; - try { - response = clientSupplier.get().queryOffset(this.name, this.stream); - } catch (Exception e) { - throw new IllegalStateException( - format( - "Not possible to query offset for consumer %s on stream %s for now: %s", - this.name, this.stream, e.getMessage()), - e); - } - if (response.isOk()) { - return response.getOffset(); - } else if (response.getResponseCode() == Constants.RESPONSE_CODE_NO_OFFSET) { - throw new NoOffsetException( - format( - "No offset stored for consumer %s on stream %s (%s)", - this.name, this.stream, Utils.formatConstant(response.getResponseCode()))); - } else { - throw new StreamException( - format( - "QueryOffset for consumer %s on stream %s returned an error (%s)", - this.name, this.stream, Utils.formatConstant(response.getResponseCode())), - response.getResponseCode()); - } + return OffsetTrackingUtils.storedOffset(clientSupplier, this.name, this.stream); } else if (this.name == null) { throw new UnsupportedOperationException( "Not possible to query stored offset for a consumer without a name"); diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java index eb3e0cae77..ec00136800 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java @@ -18,6 +18,7 @@ import static com.rabbitmq.stream.impl.Utils.SUBSCRIPTION_PROPERTY_MATCH_UNFILTERED; import com.rabbitmq.stream.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.time.Duration; @@ -112,6 +113,7 @@ public ConsumerBuilder subscriptionListener(SubscriptionListener subscriptionLis } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public ManualTrackingStrategy manualTrackingStrategy() { this.manualTrackingStrategy = new DefaultManualTrackingStrategy(this); this.autoTrackingStrategy = null; @@ -120,6 +122,7 @@ public ManualTrackingStrategy manualTrackingStrategy() { } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public AutoTrackingStrategy autoTrackingStrategy() { this.autoTrackingStrategy = new DefaultAutoTrackingStrategy(this); this.manualTrackingStrategy = null; @@ -128,6 +131,7 @@ public AutoTrackingStrategy autoTrackingStrategy() { } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public ConsumerBuilder noTrackingStrategy() { this.noTrackingStrategy = true; this.autoTrackingStrategy = null; @@ -140,6 +144,7 @@ public FlowConfiguration flow() { return this.flowConfiguration; } + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") StreamConsumerBuilder lazyInit(boolean lazyInit) { this.lazyInit = lazyInit; return this; diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java index 1aa1083f78..ad41eccbaf 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java @@ -38,7 +38,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBufAllocator; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import java.io.IOException; @@ -134,8 +133,6 @@ class StreamEnvironment implements Environment { : tlsConfiguration.sslContext(); clientParametersPrototype.sslContext(sslContext); - clientParametersPrototype.tlsHostnameVerification( - tlsConfiguration.hostnameVerificationEnabled()); } catch (SSLException e) { throw new StreamException("Error while creating Netty SSL context", e); @@ -207,122 +204,136 @@ class StreamEnvironment implements Environment { .collect(toList()); this.locators = List.copyOf(lctrs); - this.executorServiceFactory = - new DefaultExecutorServiceFactory( - this.addresses.size(), 1, "rabbitmq-stream-locator-connection-"); - - if (clientParametersPrototype.eventLoopGroup == null) { - this.eventLoopGroup = new NioEventLoopGroup(); - this.clientParametersPrototype = - clientParametersPrototype.duplicate().eventLoopGroup(this.eventLoopGroup); - } else { - this.eventLoopGroup = null; - this.clientParametersPrototype = - clientParametersPrototype - .duplicate() - .eventLoopGroup(clientParametersPrototype.eventLoopGroup); - } - ScheduledExecutorService executorService; - if (scheduledExecutorService == null) { - int threads = Runtime.getRuntime().availableProcessors(); - LOGGER.debug("Creating scheduled executor service with {} thread(s)", threads); - ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-scheduler-"); - executorService = Executors.newScheduledThreadPool(threads, threadFactory); - this.privateScheduleExecutorService = true; - } else { - executorService = scheduledExecutorService; - this.privateScheduleExecutorService = false; - } - this.scheduledExecutorService = executorService; - - this.producersCoordinator = - new ProducersCoordinator( - this, - maxProducersByConnection, - maxTrackingConsumersByConnection, - connectionNamingStrategy, - coordinatorClientFactory(this, producerNodeRetryDelay), - forceLeaderForProducers); - this.consumersCoordinator = - new ConsumersCoordinator( - this, - maxConsumersByConnection, - connectionNamingStrategy, - coordinatorClientFactory(this, consumerNodeRetryDelay), - forceReplicaForConsumers, - Utils.brokerPicker()); - this.offsetTrackingCoordinator = new OffsetTrackingCoordinator(this); - - ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-locator-scheduler-"); - this.locatorReconnectionScheduledExecutorService = - Executors.newScheduledThreadPool(this.locators.size(), threadFactory); - - ClientParameters clientParametersForInit = locatorParametersCopy(); - Runnable locatorInitSequence = - () -> { - RuntimeException lastException = null; - for (int i = 0; i < locators.size(); i++) { - Address address = addresses.get(i % addresses.size()); - Locator locator = locator(i); - address = addressResolver.resolve(address); - String connectionName = connectionNamingStrategy.apply(ClientConnectionType.LOCATOR); - Client.ClientParameters locatorParameters = - clientParametersForInit - .duplicate() - .host(address.host()) - .port(address.port()) - .clientProperty("connection_name", connectionName) - .shutdownListener( - shutdownListener(locator, connectionNamingStrategy, clientFactory)); - try { - Client client = clientFactory.apply(locatorParameters); - locator.client(client); - LOGGER.debug("Created locator connection '{}'", connectionName); - LOGGER.debug("Locator connected to {}", address); - } catch (RuntimeException e) { - LOGGER.debug("Error while try to connect to {}: {}", address, e.getMessage()); - lastException = e; + ShutdownService shutdownService = new ShutdownService(); + try { + this.executorServiceFactory = + new DefaultExecutorServiceFactory( + this.addresses.size(), 1, "rabbitmq-stream-locator-connection-"); + shutdownService.wrap(this.executorServiceFactory::close); + + if (clientParametersPrototype.eventLoopGroup == null) { + this.eventLoopGroup = Utils.eventLoopGroup(); + shutdownService.wrap(() -> closeEventLoopGroup(this.eventLoopGroup)); + this.clientParametersPrototype = + clientParametersPrototype.duplicate().eventLoopGroup(this.eventLoopGroup); + } else { + this.eventLoopGroup = null; + this.clientParametersPrototype = + clientParametersPrototype + .duplicate() + .eventLoopGroup(clientParametersPrototype.eventLoopGroup); + } + ScheduledExecutorService executorService; + if (scheduledExecutorService == null) { + int threads = AVAILABLE_PROCESSORS; + LOGGER.debug("Creating scheduled executor service with {} thread(s)", threads); + ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-scheduler-"); + executorService = Executors.newScheduledThreadPool(threads, threadFactory); + shutdownService.wrap(executorService::shutdownNow); + this.privateScheduleExecutorService = true; + } else { + executorService = scheduledExecutorService; + this.privateScheduleExecutorService = false; + } + this.scheduledExecutorService = executorService; + + this.producersCoordinator = + new ProducersCoordinator( + this, + maxProducersByConnection, + maxTrackingConsumersByConnection, + connectionNamingStrategy, + coordinatorClientFactory(this, producerNodeRetryDelay), + forceLeaderForProducers); + shutdownService.wrap(this.producersCoordinator::close); + this.consumersCoordinator = + new ConsumersCoordinator( + this, + maxConsumersByConnection, + connectionNamingStrategy, + coordinatorClientFactory(this, consumerNodeRetryDelay), + forceReplicaForConsumers, + Utils.brokerPicker()); + shutdownService.wrap(this.consumersCoordinator::close); + this.offsetTrackingCoordinator = new OffsetTrackingCoordinator(this); + shutdownService.wrap(this.offsetTrackingCoordinator::close); + + ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-locator-scheduler-"); + this.locatorReconnectionScheduledExecutorService = + Executors.newScheduledThreadPool(this.locators.size(), threadFactory); + shutdownService.wrap(this.locatorReconnectionScheduledExecutorService::shutdownNow); + + ClientParameters clientParametersForInit = locatorParametersCopy(); + Runnable locatorInitSequence = + () -> { + RuntimeException lastException = null; + for (int i = 0; i < locators.size(); i++) { + Address address = addresses.get(i % addresses.size()); + Locator locator = locator(i); + address = addressResolver.resolve(address); + String connectionName = connectionNamingStrategy.apply(ClientConnectionType.LOCATOR); + Client.ClientParameters locatorParameters = + clientParametersForInit + .duplicate() + .host(address.host()) + .port(address.port()) + .clientProperty("connection_name", connectionName) + .shutdownListener( + shutdownListener(locator, connectionNamingStrategy, clientFactory)); + try { + Client client = clientFactory.apply(locatorParameters); + locator.client(client); + LOGGER.debug("Created locator connection '{}'", connectionName); + LOGGER.debug("Locator connected to {}", address); + } catch (RuntimeException e) { + LOGGER.debug("Error while try to connect to {}: {}", address, e.getMessage()); + lastException = e; + } } - } - if (this.locators.stream().allMatch(Locator::isNotSet)) { - throw lastException == null - ? new StreamException("Not locator available") - : lastException; - } else { - this.locators.forEach( - l -> { - if (l.isNotSet()) { - ShutdownListener shutdownListener = - shutdownListener(l, connectionNamingStrategy, clientFactory); - Client.ClientParameters newLocatorParameters = - this.locatorParametersCopy().shutdownListener(shutdownListener); - scheduleLocatorConnection( - newLocatorParameters, - this.addressResolver, - l, - connectionNamingStrategy, - clientFactory, - this.locatorReconnectionScheduledExecutorService, - this.recoveryBackOffDelayPolicy, - l.label()); - } - }); - } - }; - if (lazyInit) { - this.locatorInitializationSequence = locatorInitSequence; - } else { - locatorInitSequence.run(); - locatorsInitialized.set(true); - this.locatorInitializationSequence = () -> {}; + if (this.locators.stream().allMatch(Locator::isNotSet)) { + throw lastException == null + ? new StreamException("Not locator available") + : lastException; + } else { + this.locators.forEach( + l -> { + if (l.isNotSet()) { + ShutdownListener shutdownListener = + shutdownListener(l, connectionNamingStrategy, clientFactory); + Client.ClientParameters newLocatorParameters = + this.locatorParametersCopy().shutdownListener(shutdownListener); + scheduleLocatorConnection( + newLocatorParameters, + this.addressResolver, + l, + connectionNamingStrategy, + clientFactory, + this.locatorReconnectionScheduledExecutorService, + this.recoveryBackOffDelayPolicy, + l.label()); + } + }); + } + }; + if (lazyInit) { + this.locatorInitializationSequence = locatorInitSequence; + } else { + locatorInitSequence.run(); + locatorsInitialized.set(true); + this.locatorInitializationSequence = () -> {}; + } + this.codec = + clientParametersPrototype.codec() == null + ? Codecs.DEFAULT + : clientParametersPrototype.codec(); + this.clockRefreshFuture = + this.scheduledExecutorService.scheduleAtFixedRate( + namedRunnable(this.clock::refresh, "Background clock refresh"), 1, 1, SECONDS); + shutdownService.wrap(() -> this.clockRefreshFuture.cancel(false)); + } catch (RuntimeException e) { + shutdownService.close(); + throw e; } - this.codec = - clientParametersPrototype.codec() == null - ? Codecs.DEFAULT - : clientParametersPrototype.codec(); - this.clockRefreshFuture = - this.scheduledExecutorService.scheduleAtFixedRate( - namedRunnable(this.clock::refresh, "Background clock refresh"), 1, 1, SECONDS); } private ShutdownListener shutdownListener( @@ -559,6 +570,29 @@ public StreamStats queryStreamStats(String stream) { } } + @Override + public void storeOffset(String reference, String stream, long offset) { + checkNotClosed(); + this.maybeInitializeLocator(); + locatorOperation( + Utils.namedFunction( + l -> { + l.storeOffset(reference, stream, offset); + return null; + }, + "Store offset %d for stream '%s' with reference '%s'", + offset, + stream, + reference)); + OffsetTrackingUtils.waitForOffsetToBeStored( + "env-store-offset", + this.scheduledExecutorService, + () -> OffsetTrackingUtils.storedOffset(() -> locator().client(), reference, stream), + reference, + stream, + offset); + } + @Override public boolean streamExists(String stream) { checkNotClosed(); @@ -697,20 +731,24 @@ public void close() { if (this.locatorReconnectionScheduledExecutorService != null) { this.locatorReconnectionScheduledExecutorService.shutdownNow(); } - try { - if (this.eventLoopGroup != null - && (!this.eventLoopGroup.isShuttingDown() || !this.eventLoopGroup.isShutdown())) { - LOGGER.debug("Closing Netty event loop group"); - this.eventLoopGroup.shutdownGracefully(1, 10, SECONDS).get(10, SECONDS); - } - } catch (InterruptedException e) { - LOGGER.info("Event loop group closing has been interrupted"); - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - LOGGER.info("Event loop group closing failed", e); - } catch (TimeoutException e) { - LOGGER.info("Could not close event loop group in 10 seconds"); + closeEventLoopGroup(this.eventLoopGroup); + } + } + + private static void closeEventLoopGroup(EventLoopGroup eventLoopGroup) { + try { + if (eventLoopGroup != null + && (!eventLoopGroup.isShuttingDown() || !eventLoopGroup.isShutdown())) { + LOGGER.debug("Closing Netty event loop group"); + eventLoopGroup.shutdownGracefully(1, 10, SECONDS).get(10, SECONDS); } + } catch (InterruptedException e) { + LOGGER.info("Event loop group closing has been interrupted"); + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + LOGGER.info("Event loop group closing failed", e); + } catch (TimeoutException e) { + LOGGER.info("Could not close event loop group in 10 seconds"); } } diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java index 555200dee0..1be66aa36b 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java @@ -372,18 +372,6 @@ private DefaultTlsConfiguration(EnvironmentBuilder environmentBuilder) { this.environmentBuilder = environmentBuilder; } - @Override - public TlsConfiguration hostnameVerification() { - this.hostnameVerification = true; - return this; - } - - @Override - public TlsConfiguration hostnameVerification(boolean hostnameVerification) { - this.hostnameVerification = hostnameVerification; - return this; - } - @Override public TlsConfiguration sslContext(SslContext sslContext) { this.sslContext = sslContext; @@ -400,6 +388,7 @@ public TlsConfiguration trustEverything() { this.sslContext( SslContextBuilder.forClient() .trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER) + .endpointIdentificationAlgorithm(null) .build()); } catch (SSLException e) { throw new StreamException("Error while creating Netty SSL context", e); @@ -433,7 +422,7 @@ static class DefaultNettyConfiguration implements NettyConfiguration { private final EnvironmentBuilder environmentBuilder; private EventLoopGroup eventLoopGroup; - private ByteBufAllocator byteBufAllocator = ByteBufAllocator.DEFAULT; + private ByteBufAllocator byteBufAllocator = Utils.byteBufAllocator(); private Consumer channelCustomizer = noOpConsumer(); private Consumer bootstrapCustomizer = noOpConsumer(); diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java b/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java index 27552512c8..516afe16f3 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java @@ -180,6 +180,7 @@ public int fragmentLength(Object entity) { dynamicBatch, subEntrySize, batchSize, + maxUnconfirmedMessages, compressionCodec, environment.codec(), environment.byteBufAllocator(), @@ -189,7 +190,8 @@ public int fragmentLength(Object entity) { environment.clock(), stream, environment.observationCollector(), - this); + this, + this.id); boolean backgroundBatchPublishingTaskRequired = !dynamicBatch && batchPublishingDelay.toMillis() > 0; diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java index 43a57bbc5c..b44290868c 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java @@ -28,8 +28,8 @@ class StreamProducerBuilder implements ProducerBuilder { - static final boolean DEFAULT_DYNAMIC_BATCH = true; -// Boolean.parseBoolean(System.getProperty("rabbitmq.stream.producer.dynamic.batch", "true")); + static final boolean DEFAULT_DYNAMIC_BATCH = + Boolean.parseBoolean(System.getProperty("rabbitmq.stream.producer.dynamic.batch", "true")); private final StreamEnvironment environment; @@ -201,7 +201,7 @@ public Producer build() { if (this.routingConfiguration == null && this.superStream != null) { throw new IllegalArgumentException( - "A routing configuration must specified when a super stream is set"); + "A routing configuration must be specified when a super stream is set"); } if (this.stream != null) { diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java b/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java index 86e3693ad7..c3d216046e 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java @@ -32,7 +32,7 @@ class StreamStreamCreator implements StreamCreator { private final StreamEnvironment environment; private final Client.StreamParametersBuilder streamParametersBuilder = - new Client.StreamParametersBuilder().leaderLocator(LeaderLocator.LEAST_LEADERS); + new Client.StreamParametersBuilder().leaderLocator(LeaderLocator.BALANCED); private String name; private DefaultSuperStreamConfiguration superStreamConfiguration; diff --git a/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java b/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java index 4d73c19123..a8903b7924 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java +++ b/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java @@ -17,11 +17,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,48 +29,30 @@ final class ThreadUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ThreadUtils.class); private static final ThreadFactory THREAD_FACTORY; - private static final Function EXECUTOR_SERVICE_FACTORY; private static final Predicate IS_VIRTUAL; static { + ThreadFactory tf; if (isJava21OrMore()) { LOGGER.debug("Running Java 21 or more, using virtual threads"); - Class builderClass = - Arrays.stream(Thread.class.getDeclaredClasses()) - .filter(c -> "Builder".equals(c.getSimpleName())) - .findFirst() - .get(); - // Reflection code is the same as: - // Thread.ofVirtual().factory(); try { + Class builderClass = + Arrays.stream(Thread.class.getDeclaredClasses()) + .filter(c -> "Builder".equals(c.getSimpleName())) + .findFirst() + .get(); + // Reflection code is the same as: + // Thread.ofVirtual().factory(); Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); - THREAD_FACTORY = (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - throw new RuntimeException(e); + tf = (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); + } catch (IllegalAccessException + | InvocationTargetException + | NoSuchMethodException + | RuntimeException e) { + LOGGER.debug("Error when creating virtual thread factory on Java 21+: {}", e.getMessage()); + LOGGER.debug("Falling back to default thread factory"); + tf = Executors.defaultThreadFactory(); } - EXECUTOR_SERVICE_FACTORY = - prefix -> { - try { - // Reflection code is the same as the 2 following lines: - // ThreadFactory factory = Thread.ofVirtual().name(prefix, 0).factory(); - // Executors.newThreadPerTaskExecutor(factory); - Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); - if (prefix != null) { - builder = - builderClass - .getDeclaredMethod("name", String.class, Long.TYPE) - .invoke(builder, prefix, 0L); - } - ThreadFactory factory = - (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); - return (ExecutorService) - Executors.class - .getDeclaredMethod("newThreadPerTaskExecutor", ThreadFactory.class) - .invoke(null, factory); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - throw new RuntimeException(e); - } - }; IS_VIRTUAL = thread -> { Method method = null; @@ -85,14 +65,21 @@ final class ThreadUtils { } }; } else { - THREAD_FACTORY = Executors.defaultThreadFactory(); - EXECUTOR_SERVICE_FACTORY = prefix -> Executors.newCachedThreadPool(threadFactory(prefix)); + tf = Executors.defaultThreadFactory(); IS_VIRTUAL = ignored -> false; } + THREAD_FACTORY = tf; } private ThreadUtils() {} + /** + * Create a thread factory that prefixes thread names. Based on {@link + * java.util.concurrent.Executors#defaultThreadFactory()}, so always creates platform threads. + * + * @param prefix used to prefix thread names + * @return thread factory + */ static ThreadFactory threadFactory(String prefix) { if (prefix == null) { return Executors.defaultThreadFactory(); @@ -101,8 +88,31 @@ static ThreadFactory threadFactory(String prefix) { } } + /** + * Returns a thread factory that creates virtual threads if available. + * + * @param prefix + * @return + */ static ThreadFactory internalThreadFactory(String prefix) { - return new NamedThreadFactory(THREAD_FACTORY, prefix); + if (prefix == null) { + return THREAD_FACTORY; + } else { + return new NamedThreadFactory(THREAD_FACTORY, prefix); + } + } + + /** + * Creates a virtual thread if available. + * + * @param name + * @param task + * @return + */ + static Thread newInternalThread(String name, Runnable task) { + Thread t = THREAD_FACTORY.newThread(task); + t.setName(name); + return t; } static boolean isVirtual(Thread thread) { diff --git a/src/main/java/com/rabbitmq/stream/impl/Utils.java b/src/main/java/com/rabbitmq/stream/impl/Utils.java index d00728954f..6a4b08ebd5 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Utils.java +++ b/src/main/java/com/rabbitmq/stream/impl/Utils.java @@ -19,7 +19,12 @@ import com.rabbitmq.stream.*; import com.rabbitmq.stream.impl.Client.ClientParameters; +import io.netty.buffer.ByteBufAllocator; import io.netty.channel.ConnectTimeoutException; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.nio.NioIoHandler; import java.net.UnknownHostException; import java.security.cert.X509Certificate; import java.time.Duration; @@ -41,6 +46,8 @@ final class Utils { + static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors(); + @SuppressWarnings("rawtypes") private static final Consumer NO_OP_CONSUMER = o -> {}; @@ -55,6 +62,8 @@ final class Utils { static final String SUBSCRIPTION_PROPERTY_FILTER_PREFIX = "filter."; static final String SUBSCRIPTION_PROPERTY_MATCH_UNFILTERED = "match-unfiltered"; + static final boolean IS_NETTY_4_2; + static { Map labels = new HashMap<>(); Arrays.stream(Constants.class.getDeclaredFields()) @@ -70,6 +79,14 @@ final class Utils { } }); CONSTANT_LABELS = copyOf(labels); + + boolean netty4_2 = true; + try { + Class.forName("io.netty.channel.MultiThreadIoEventLoopGroup"); + } catch (ClassNotFoundException e) { + netty4_2 = false; + } + IS_NETTY_4_2 = netty4_2; } static final AddressResolver DEFAULT_ADDRESS_RESOLVER = address -> address; @@ -406,6 +423,19 @@ static Function defaultConnectionNamingStrategy(St prefixes.get(clientConnectionType) + sequences.get(clientConnectionType).getAndIncrement(); } + @SuppressWarnings("deprecation") + static EventLoopGroup eventLoopGroup() { + if (IS_NETTY_4_2) { + return new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); + } else { + return new NioEventLoopGroup(); + } + } + + static ByteBufAllocator byteBufAllocator() { + return ByteBufAllocator.DEFAULT; + } + /* class to help testing SAC on super streams */ diff --git a/src/test/java/SanityCheck.java b/src/test/java/SanityCheck.java index 4e550c8bd2..7ee907180c 100755 --- a/src/test/java/SanityCheck.java +++ b/src/test/java/SanityCheck.java @@ -1,5 +1,4 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//REPOS mavencentral,ossrh-staging=https://oss.sonatype.org/content/groups/staging/,rabbitmq-packagecloud-milestones=https://packagecloud.io/rabbitmq/maven-milestones/maven2 //DEPS com.rabbitmq:stream-client:${env.RABBITMQ_LIBRARY_VERSION} //DEPS org.slf4j:slf4j-simple:1.7.36 diff --git a/src/test/java/com/rabbitmq/stream/Cli.java b/src/test/java/com/rabbitmq/stream/Cli.java index 7097725af4..26f54901ac 100644 --- a/src/test/java/com/rabbitmq/stream/Cli.java +++ b/src/test/java/com/rabbitmq/stream/Cli.java @@ -178,6 +178,30 @@ static List toConnectionInfoList(String json) { return GSON.fromJson(json, new TypeToken>() {}.getType()); } + public static List listGroupConsumers(String stream, String reference) { + ProcessState process = + rabbitmqStreams( + format( + "list_stream_group_consumers -q --stream %s --reference %s " + + "--formatter table subscription_id,state", + stream, reference)); + + List itemList = Collections.emptyList(); + String content = process.output(); + String[] lines = content.split(System.lineSeparator()); + if (lines.length > 1) { + itemList = new ArrayList<>(lines.length - 1); + for (int i = 1; i < lines.length; i++) { + String line = lines[i]; + String[] fields = line.split("\t"); + String id = fields[0]; + String state = fields[1].replace("\"", ""); + itemList.add(new SubscriptionInfo(Integer.parseInt(id), state)); + } + } + return itemList; + } + public static void restartStream(String stream) { rabbitmqStreams(" restart_stream " + stream); } @@ -236,10 +260,7 @@ public static void setEnv(String parameter, String value) { } public static String rabbitmqctlCommand() { - String rabbitmqCtl = System.getProperty("rabbitmqctl.bin"); - if (rabbitmqCtl == null) { - rabbitmqCtl = DOCKER_PREFIX + "rabbitmq"; - } + String rabbitmqCtl = rabbitmqctlBin(); if (rabbitmqCtl.startsWith(DOCKER_PREFIX)) { String containerId = rabbitmqCtl.split(":")[1]; return "docker exec " + containerId + " rabbitmqctl"; @@ -248,6 +269,14 @@ public static String rabbitmqctlCommand() { } } + private static String rabbitmqctlBin() { + String rabbitmqCtl = System.getProperty("rabbitmqctl.bin"); + if (rabbitmqCtl == null) { + rabbitmqCtl = DOCKER_PREFIX + "rabbitmq"; + } + return rabbitmqCtl; + } + private static String rabbitmqStreamsCommand() { String rabbitmqctl = rabbitmqctlCommand(); int lastIndex = rabbitmqctl.lastIndexOf("rabbitmqctl"); @@ -306,11 +335,7 @@ private static void clearResourceAlarm(String source) { } public static boolean isOnDocker() { - String rabbitmqCtl = System.getProperty("rabbitmqctl.bin"); - if (rabbitmqCtl == null) { - throw new IllegalStateException("Please define the rabbitmqctl.bin system property"); - } - return rabbitmqCtl.startsWith(DOCKER_PREFIX); + return rabbitmqctlBin().startsWith(DOCKER_PREFIX); } public static List nodes() { @@ -420,6 +445,30 @@ public String toString() { } } + public static final class SubscriptionInfo { + + private final int id; + private final String state; + + public SubscriptionInfo(int id, String state) { + this.id = id; + this.state = state; + } + + public int id() { + return this.id; + } + + public String state() { + return this.state; + } + + @Override + public String toString() { + return "SubscriptionInfo{id='" + id + '\'' + ", state='" + state + '\'' + '}'; + } + } + public static class ProcessState { private final InputStreamPumpState inputState; diff --git a/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java b/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java index 4c45d6ae44..0441f5e6a9 100644 --- a/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java +++ b/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java @@ -19,7 +19,8 @@ import com.rabbitmq.stream.impl.Client; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.nio.NioIoHandler; import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -31,7 +32,7 @@ public class DefaultEnvironmentTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/StreamClientTestExtension.java b/src/test/java/com/rabbitmq/stream/StreamClientTestExtension.java new file mode 100644 index 0000000000..913801170b --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/StreamClientTestExtension.java @@ -0,0 +1,65 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream; + +import java.util.ArrayList; +import java.util.Collection; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamClientTestExtension implements BeforeEachCallback, AfterEachCallback { + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamClientTestExtension.class); + private static final ExtensionContext.Namespace NS = + ExtensionContext.Namespace.create(StreamClientTestExtension.class); + + @Override + public void beforeEach(ExtensionContext ctx) { + ExtensionContext.Store store = store(ctx); + store.put("threads", threads()); + } + + @Override + public void afterEach(ExtensionContext ctx) { + @SuppressWarnings("unchecked") + Collection initialThreads = (Collection) store(ctx).remove("threads"); + if (initialThreads != null) { + Collection threads = threads(); + if (threads.size() > initialThreads.size()) { + Collection diff = new ArrayList<>(threads); + diff.removeAll(initialThreads); + LOGGER.warn( + "[{}] There should be no new threads, initial {}, current {} (diff: {})", + ctx.getTestMethod().get().getName(), + initialThreads.size(), + threads.size(), + diff); + } + } else { + LOGGER.warn("No threads in test context"); + } + } + + private static ExtensionContext.Store store(ExtensionContext ctx) { + return ctx.getStore(NS); + } + + private Collection threads() { + return Thread.getAllStackTraces().keySet(); + } +} diff --git a/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java b/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java index 064e013617..b80acf1b2f 100644 --- a/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java +++ b/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java @@ -33,18 +33,18 @@ import java.math.BigInteger; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Stream; +import org.apache.qpid.proton.amqp.Symbol; import org.apache.qpid.proton.amqp.messaging.AmqpValue; +import org.apache.qpid.proton.amqp.messaging.MessageAnnotations; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.ThrowableAssert; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -248,6 +248,22 @@ void codecs(CodecCouple codecCouple) { .entry("annotations.string", string) .entrySymbol("annotations.symbol", symbol) .entry("annotations.null", (String) null) + .entry( + "list", + List.of("1", "2", 3, List.of("1"), Map.of("k1", "v1"), new String[] {"1"})) + .entry( + "map", + Map.of( + "k1", + "v1", + "k2", + List.of("v2"), + "k3", + Map.of("k1", "v1"), + "k4", + new String[] {"1"})) + .entryArray("arrayString", new String[] {"1", "2", "3"}) + .entryArray("arrayInt", new Integer[] {200, 201, 202}) .messageBuilder() .build(); outboundMessage.annotate("extra.annotation", "extra annotation value"); @@ -474,6 +490,35 @@ void codecs(CodecCouple codecCouple) { .isNotNull() .isInstanceOf(String.class) .isEqualTo("extra annotation value"); + + List list = (List) inboundMessage.getMessageAnnotations().get("list"); + assertThat(list.get(0)).isEqualTo("1"); + assertThat(list.get(1)).isEqualTo("2"); + assertThat(list.get(2)).isEqualTo(3); + assertThat(list.get(3)).isEqualTo(List.of("1")); + assertThat(list.get(4)).isEqualTo(Map.of("k1", "v1")); + assertThat(list.get(5)).isEqualTo(new String[] {"1"}); + + Map map = (Map) inboundMessage.getMessageAnnotations().get("map"); + assertThat(map.get("k1")).isEqualTo("v1"); + assertThat(map.get("k2")).isEqualTo(List.of("v2")); + assertThat(map.get("k3")).isEqualTo(Map.of("k1", "v1")); + assertThat(map.get("k4")).isEqualTo(new String[] {"1"}); + + Object[] arrayString = + (Object[]) inboundMessage.getMessageAnnotations().get("arrayString"); + assertThat(arrayString).containsExactly("1", "2", "3"); + int[] arrayInt; + // QPid codec returns int[] and SwiftMQ codec returns Integer[] + if (inboundMessage.getMessageAnnotations().get("arrayInt") instanceof Integer[]) { + arrayInt = + Arrays.stream((Integer[]) inboundMessage.getMessageAnnotations().get("arrayInt")) + .mapToInt(Integer::intValue) + .toArray(); + } else { + arrayInt = (int[]) inboundMessage.getMessageAnnotations().get("arrayInt"); + } + assertThat(arrayInt).containsExactly(200, 201, 202); }); } @@ -624,6 +669,40 @@ void copy(CodecCouple codecCouple) { .containsEntry("copy", "copy value"); } + @Test + void qpidDoesNotSupportPrimitiveArrayEncodingInMap() { + org.apache.qpid.proton.message.Message message = + org.apache.qpid.proton.message.Message.Factory.create(); + Map map = new LinkedHashMap<>(); + map.put(Symbol.valueOf("foo"), new int[] {1, 2, 3}); + message.setMessageAnnotations(new MessageAnnotations(map)); + assertThatThrownBy(() -> qpidEncodeDecode(message)).isInstanceOf(ClassCastException.class); + } + + @Test + void qpidEncodeIntegerArrayDecodeIntArrayInMap() { + org.apache.qpid.proton.message.Message message = + org.apache.qpid.proton.message.Message.Factory.create(); + Map map = new LinkedHashMap<>(); + map.put(Symbol.valueOf("foo"), new Integer[] {1, 2, 3}); + message.setMessageAnnotations(new MessageAnnotations(map)); + message = qpidEncodeDecode(message); + map = message.getMessageAnnotations().getValue(); + assertThat(map.get(Symbol.valueOf("foo"))).isInstanceOf(int[].class); + } + + private static org.apache.qpid.proton.message.Message qpidEncodeDecode( + org.apache.qpid.proton.message.Message in) { + QpidProtonCodec.ByteArrayWritableBuffer writableBuffer = + new QpidProtonCodec.ByteArrayWritableBuffer(8192); + in.encode(writableBuffer); + + org.apache.qpid.proton.message.Message out = + org.apache.qpid.proton.message.Message.Factory.create(); + out.decode(writableBuffer.getArray(), 0, writableBuffer.getArrayLength()); + return out; + } + MessageTestConfiguration test( Function messageOperation, Consumer messageExpectation) { diff --git a/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java b/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java index 4f4c001982..bd7a9c6bc2 100644 --- a/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java @@ -19,7 +19,9 @@ import com.rabbitmq.stream.observation.micrometer.MicrometerObservationCollectorBuilder; import io.micrometer.observation.ObservationRegistry; import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollIoHandler; import io.netty.channel.epoll.EpollSocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; @@ -140,7 +142,9 @@ void deleteStream() { void nativeEpoll() { // tag::native-epoll[] - EventLoopGroup epollEventLoopGroup = new EpollEventLoopGroup(); // <1> + EventLoopGroup epollEventLoopGroup = new MultiThreadIoEventLoopGroup( // <1> + EpollIoHandler.newFactory() // <1> + ); // <1> Environment environment = Environment.builder() .netty() // <2> .eventLoopGroup(epollEventLoopGroup) // <3> diff --git a/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java b/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java index 2ee819b058..78141e8130 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java @@ -32,7 +32,6 @@ import com.rabbitmq.stream.Producer; import com.rabbitmq.stream.StreamException; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.time.Duration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -56,7 +55,7 @@ public class AlarmsTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/Assertions.java b/src/test/java/com/rabbitmq/stream/impl/Assertions.java index 8a3b0e966f..10b7093774 100644 --- a/src/test/java/com/rabbitmq/stream/impl/Assertions.java +++ b/src/test/java/com/rabbitmq/stream/impl/Assertions.java @@ -16,6 +16,7 @@ import static org.assertj.core.api.Assertions.fail; +import com.rabbitmq.stream.Constants; import java.time.Duration; import org.assertj.core.api.AbstractObjectAssert; @@ -23,10 +24,50 @@ final class Assertions { private Assertions() {} + static ResponseAssert assertThat(Client.Response response) { + return new ResponseAssert(response); + } + static SyncAssert assertThat(TestUtils.Sync sync) { return new SyncAssert(sync); } + static class ResponseAssert extends AbstractObjectAssert { + + public ResponseAssert(Client.Response response) { + super(response, ResponseAssert.class); + } + + ResponseAssert isOk() { + if (!actual.isOk()) { + fail( + "Response should be successful but was not, response code is: %s", + Utils.formatConstant(actual.getResponseCode())); + } + return this; + } + + ResponseAssert isNotOk() { + if (actual.isOk()) { + fail("Response should not be successful but was, response code is: %s", actual); + } + return this; + } + + ResponseAssert hasCode(short responseCode) { + if (actual.getResponseCode() != responseCode) { + fail( + "Response code should be %s but was %s", + Utils.formatConstant(responseCode), Utils.formatConstant(actual.getResponseCode())); + } + return this; + } + + ResponseAssert hasCodeNoOffset() { + return hasCode(Constants.RESPONSE_CODE_NO_OFFSET); + } + } + static class SyncAssert extends AbstractObjectAssert { private SyncAssert(TestUtils.Sync sync) { diff --git a/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java b/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java index 7c18ded3b3..6f81a337b7 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java @@ -17,6 +17,7 @@ import static com.rabbitmq.stream.BackOffDelayPolicy.fixedWithInitialDelay; import static com.rabbitmq.stream.impl.Assertions.assertThat; import static com.rabbitmq.stream.impl.TestUtils.sync; +import static com.rabbitmq.stream.impl.ThreadUtils.internalThreadFactory; import static java.time.Duration.*; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; @@ -61,6 +62,7 @@ void callbackCalledIfCompletedImmediately(ScheduledExecutorService scheduler) th completableFuture.thenAccept(result::set); assertThat(result.get()).isEqualTo(42); verify(task, times(1)).call(); + scheduler.shutdownNow(); } @ParameterizedTest @@ -82,6 +84,7 @@ void shouldRetryWhenExecutionFails(ScheduledExecutorService scheduler) throws Ex assertThat(latch.await(1, TimeUnit.SECONDS)).isTrue(); assertThat(result.get()).isEqualTo(42); verify(task, times(3)).call(); + scheduler.shutdownNow(); } @ParameterizedTest @@ -112,6 +115,7 @@ void shouldTimeoutWhenExecutionFailsForTooLong(ScheduledExecutorService schedule assertThat(acceptCalled.get()).isFalse(); assertThat(exceptionallyCalled.get()).isTrue(); verify(task, atLeast(5)).call(); + scheduler.shutdownNow(); } @ParameterizedTest @@ -137,6 +141,7 @@ void shouldRetryWhenPredicateAllowsIt(ScheduledExecutorService scheduler) throws assertThat(latch.await(1, TimeUnit.SECONDS)).isTrue(); assertThat(result.get()).isEqualTo(42); verify(task, times(3)).call(); + scheduler.shutdownNow(); } @ParameterizedTest @@ -171,6 +176,7 @@ void shouldFailWhenPredicateDoesNotAllowRetry(ScheduledExecutorService scheduler assertThat(acceptCalled.get()).isFalse(); assertThat(exceptionallyCalled.get()).isTrue(); verify(task, times(3)).call(); + scheduler.shutdownNow(); } @ParameterizedTest @@ -193,12 +199,12 @@ void completeExceptionally(ScheduledExecutorService scheduler) throws Exception }, scheduler); assertThat(sync).completes(); + scheduler.shutdownNow(); } static List schedulers() { return List.of( Executors.newSingleThreadScheduledExecutor(), - Executors.newScheduledThreadPool( - 0, ThreadUtils.internalThreadFactory("async-retry-test-"))); + Executors.newScheduledThreadPool(0, internalThreadFactory("async-retry-test-"))); } } diff --git a/src/test/java/com/rabbitmq/stream/impl/ClientTest.java b/src/test/java/com/rabbitmq/stream/impl/ClientTest.java index 34496ac73d..9e04942d17 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ClientTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ClientTest.java @@ -38,7 +38,7 @@ import com.rabbitmq.stream.impl.TestUtils.DisabledIfFilteringNotSupported; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.ChannelOption; import io.netty.channel.ConnectTimeoutException; import java.io.ByteArrayOutputStream; @@ -510,7 +510,7 @@ void consume() throws Exception { @ParameterizedTest @ValueSource(booleans = {true, false}) void publishAndConsume(boolean directBuffer) throws Exception { - ByteBufAllocator allocator = new UnpooledByteBufAllocator(directBuffer); + ByteBufAllocator allocator = new PooledByteBufAllocator(directBuffer); int publishCount = 1_000_000; CountDownLatch consumedLatch = new CountDownLatch(publishCount); diff --git a/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java b/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java index 4844a6fdf6..75f6fe73d7 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java @@ -60,6 +60,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -74,6 +75,7 @@ public class ConsumersCoordinatorTest { private static final SubscriptionListener NO_OP_SUBSCRIPTION_LISTENER = subscriptionContext -> {}; private static final Runnable NO_OP_TRACKING_CLOSING_CALLBACK = () -> {}; + TestInfo info; @Mock StreamEnvironment environment; @Mock StreamConsumer consumer; @Mock Client locator; @@ -113,7 +115,8 @@ static Stream> disruptionArguments() { } @BeforeEach - void init() { + void init(TestInfo info) { + this.info = info; Client.ClientParameters clientParameters = new Client.ClientParameters() { @Override @@ -169,6 +172,7 @@ public Client.ClientParameters shutdownListener( @AfterEach void tearDown() throws Exception { + this.info = null; if (coordinator != null) { // just taking the opportunity to check toString() generates valid JSON MonitoringTestUtils.extract(coordinator); @@ -2155,15 +2159,16 @@ private MessageListener lastMessageListener() { return this.messageListeners.get(messageListeners.size() - 1); } - private static ScheduledExecutorService createScheduledExecutorService() { + private ScheduledExecutorService createScheduledExecutorService() { return createScheduledExecutorService(1); } - private static ScheduledExecutorService createScheduledExecutorService(int nbThreads) { + private ScheduledExecutorService createScheduledExecutorService(int nbThreads) { + ThreadFactory tf = ThreadUtils.threadFactory(info.getTestMethod().get().getName() + "-"); return new ScheduledExecutorServiceWrapper( nbThreads == 1 - ? Executors.newSingleThreadScheduledExecutor() - : Executors.newScheduledThreadPool(nbThreads)); + ? Executors.newSingleThreadScheduledExecutor(tf) + : Executors.newScheduledThreadPool(nbThreads, tf)); } private static Response responseOk() { diff --git a/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java b/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java index 02844308dd..987cf311c9 100644 --- a/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java @@ -20,7 +20,6 @@ import com.rabbitmq.stream.impl.ServerFrameHandler.DeliverVersion1FrameHandler; import com.rabbitmq.stream.metrics.NoOpMetricsCollector; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -51,7 +50,7 @@ public MessageBuilder messageBuilder() { ByteBuf generateFrameBuffer( int nbMessages, long chunkOffset, int dataSize, Iterable messages) { - ByteBuf bb = ByteBufAllocator.DEFAULT.buffer(1024); + ByteBuf bb = Utils.byteBufAllocator().buffer(1024); bb.writeShort(Utils.encodeRequestCode(Constants.COMMAND_DELIVER)) .writeShort(Constants.VERSION_1) .writeByte(1) // subscription id diff --git a/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java b/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java index dca9810762..bc76fd6f4e 100644 --- a/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java @@ -23,10 +23,14 @@ import com.rabbitmq.stream.impl.TestUtils.Sync; import java.util.Locale; import java.util.Random; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; public class DynamicBatchTest { @@ -55,7 +59,7 @@ private static void printHistogram(Histogram histogram) { } @Test - void itemAreProcessed() { + void itemAreProcessed(TestInfo info) { MetricRegistry metrics = new MetricRegistry(); Histogram batchSizeMetrics = metrics.histogram("batch-size"); int itemCount = 3000; @@ -68,7 +72,7 @@ void itemAreProcessed() { sync.down(items.size()); return true; }; - try (DynamicBatch batch = new DynamicBatch<>(action, 100)) { + try (DynamicBatch batch = new DynamicBatch<>(action, 100, 10_000, batchId(info))) { RateLimiter rateLimiter = RateLimiter.create(10000); IntStream.range(0, itemCount) .forEach( @@ -82,7 +86,7 @@ void itemAreProcessed() { } @Test - void failedProcessingIsReplayed() throws Exception { + void failedProcessingIsReplayed(TestInfo info) throws Exception { int itemCount = 10000; AtomicInteger collected = new AtomicInteger(0); AtomicInteger processed = new AtomicInteger(0); @@ -99,7 +103,7 @@ void failedProcessingIsReplayed() throws Exception { } return result; }; - try (DynamicBatch batch = new DynamicBatch<>(action, 100)) { + try (DynamicBatch batch = new DynamicBatch<>(action, 100, 10_000, batchId(info))) { int firstRoundCount = itemCount / 5; IntStream.range(0, firstRoundCount) .forEach( @@ -118,4 +122,41 @@ void failedProcessingIsReplayed() throws Exception { waitAtMost(() -> collected.get() == itemCount); } } + + @Test + void lowThrottlingValueShouldStillHighPublishingRate(TestInfo info) throws Exception { + int batchSize = 10; + Semaphore semaphore = new Semaphore(batchSize); + DynamicBatch.BatchConsumer action = + items -> { + semaphore.release(items.size()); + return true; + }; + + try (DynamicBatch batch = new DynamicBatch<>(action, batchSize, 10_000, batchId(info))) { + MetricRegistry metrics = new MetricRegistry(); + Meter rate = metrics.meter("publishing-rate"); + AtomicBoolean keepGoing = new AtomicBoolean(true); + AtomicLong sequence = new AtomicLong(); + new Thread( + () -> { + while (keepGoing.get() && !Thread.interrupted()) { + long id = sequence.getAndIncrement(); + if (semaphore.tryAcquire()) { + batch.add(id); + rate.mark(); + } + } + }) + .start(); + long start = System.nanoTime(); + waitAtMost( + () -> + System.nanoTime() - start > TimeUnit.SECONDS.toNanos(1) && rate.getMeanRate() > 1000); + } + } + + private static String batchId(TestInfo info) { + return info.getTestMethod().get().getName(); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/FrameTest.java b/src/test/java/com/rabbitmq/stream/impl/FrameTest.java index 28422d40ad..7198facf5a 100644 --- a/src/test/java/com/rabbitmq/stream/impl/FrameTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/FrameTest.java @@ -26,7 +26,6 @@ import com.rabbitmq.stream.Message; import com.rabbitmq.stream.Properties; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import java.io.ByteArrayOutputStream; @@ -149,7 +148,7 @@ public TestDesc(String description, List sizes, int expectedCalls) { tests.forEach( test -> { Channel channel = Mockito.mock(Channel.class); - Mockito.when(channel.alloc()).thenReturn(ByteBufAllocator.DEFAULT); + Mockito.when(channel.alloc()).thenReturn(Utils.byteBufAllocator()); Mockito.when(channel.writeAndFlush(Mockito.any())) .thenReturn(Mockito.mock(ChannelFuture.class)); diff --git a/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java index 1cd14bdabd..9ee3e5b30c 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java @@ -27,7 +27,6 @@ import com.rabbitmq.stream.OffsetSpecification; import com.rabbitmq.stream.amqp.UnsignedByte; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.nio.charset.StandardCharsets; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -57,7 +56,7 @@ public class MqttInteroperabilityTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java b/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java index 9cc312b33a..e234d723c4 100644 --- a/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java @@ -16,18 +16,23 @@ import static com.rabbitmq.stream.impl.Assertions.assertThat; import static com.rabbitmq.stream.impl.LoadBalancerClusterTest.LOAD_BALANCER_ADDRESS; +import static com.rabbitmq.stream.impl.TestUtils.BrokerVersion.RABBITMQ_4_1_2; import static com.rabbitmq.stream.impl.TestUtils.newLoggerLevel; import static com.rabbitmq.stream.impl.TestUtils.sync; +import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; import static com.rabbitmq.stream.impl.Tuples.pair; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.stream; import ch.qos.logback.classic.Level; import com.google.common.collect.Streams; import com.google.common.util.concurrent.RateLimiter; import com.rabbitmq.stream.*; +import com.rabbitmq.stream.impl.TestUtils.BrokerVersionAtLeast; +import com.rabbitmq.stream.impl.TestUtils.DisabledIfNotCluster; import com.rabbitmq.stream.impl.TestUtils.Sync; import com.rabbitmq.stream.impl.Tuples.Pair; import io.netty.channel.ChannelOption; @@ -40,19 +45,24 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@TestUtils.DisabledIfNotCluster +@DisabledIfNotCluster @StreamTestInfrastructure public class RecoveryClusterTest { @@ -87,7 +97,7 @@ static void initAll() { @BeforeEach void init(TestInfo info) { - int availableProcessors = Runtime.getRuntime().availableProcessors(); + int availableProcessors = Utils.AVAILABLE_PROCESSORS; LOGGER.info("Available processors: {}", availableProcessors); ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-scheduler-"); scheduledExecutorService = Executors.newScheduledThreadPool(availableProcessors, threadFactory); @@ -134,7 +144,7 @@ void clusterRestart(boolean useLoadBalancer, boolean forceLeader) throws Interru "Cluster restart test, use load balancer {}, force leader {}", useLoadBalancer, forceLeader); - int streamCount = 10; + int streamCount = Utils.AVAILABLE_PROCESSORS; int producerCount = streamCount * 2; int consumerCount = streamCount * 2; @@ -200,15 +210,7 @@ void clusterRestart(boolean useLoadBalancer, boolean forceLeader) throws Interru syncs = consumers.stream().map(c -> c.waitForNewMessages(100)).collect(toList()); syncs.forEach(s -> assertThat(s).completes()); - nodes.forEach( - n -> { - LOGGER.info("Restarting node {}...", n); - Cli.restartNode(n); - LOGGER.info("Restarted node {}.", n); - }); - LOGGER.info("Rebalancing..."); - Cli.rebalance(); - LOGGER.info("Rebalancing over."); + restartCluster(); Thread.sleep(BACK_OFF_DELAY_POLICY.delay(0).toMillis()); @@ -290,8 +292,132 @@ void clusterRestart(boolean useLoadBalancer, boolean forceLeader) throws Interru } } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @BrokerVersionAtLeast(RABBITMQ_4_1_2) + void sacWithClusterRestart(boolean superStream) throws Exception { + environment = + environmentBuilder + .uris(URIS) + .netty() + .bootstrapCustomizer( + b -> { + b.option( + ChannelOption.CONNECT_TIMEOUT_MILLIS, + (int) BACK_OFF_DELAY_POLICY.delay(0).toMillis()); + }) + .environmentBuilder() + .maxConsumersByConnection(1) + .build(); + + int consumerCount = 3; + AtomicLong lastOffset = new AtomicLong(0); + String app = "app-name"; + String s = TestUtils.streamName(testInfo); + ProducerState pState = null; + List consumers = Collections.emptyList(); + try { + StreamCreator sCreator = environment.streamCreator().stream(s); + if (superStream) { + sCreator = sCreator.superStream().partitions(1).creator(); + } + sCreator.create(); + + pState = new ProducerState(s, true, superStream, environment); + pState.start(); + + Map consumerStatus = new ConcurrentHashMap<>(); + consumers = + IntStream.range(0, consumerCount) + .mapToObj( + i -> + new ConsumerState( + s, + environment, + b -> { + b.singleActiveConsumer() + .name(app) + .noTrackingStrategy() + .consumerUpdateListener( + ctx -> { + consumerStatus.put(i, ctx.isActive()); + return OffsetSpecification.offset(lastOffset.get()); + }); + if (superStream) { + b.superStream(s); + } else { + b.stream(s); + } + }, + (ctx, m) -> lastOffset.set(ctx.offset()))) + .collect(toList()); + + Sync sync = pState.waitForNewMessages(100); + assertThat(sync).completes(); + sync = consumers.get(0).waitForNewMessages(100); + assertThat(sync).completes(); + + String streamArg = superStream ? s + "-0" : s; + + Callable checkConsumers = + () -> { + waitAtMost( + () -> { + List subscriptions = Cli.listGroupConsumers(streamArg, app); + LOGGER.info("Group consumers: {}", subscriptions); + return subscriptions.size() == consumerCount + && subscriptions.stream() + .filter(sub -> sub.state().startsWith("active")) + .count() + == 1 + && subscriptions.stream() + .filter(sub -> sub.state().startsWith("waiting")) + .count() + == 2; + }, + () -> + "Group consumers not in expected state: " + + Cli.listGroupConsumers(streamArg, app)); + return null; + }; + + checkConsumers.call(); + + restartCluster(); + + Thread.sleep(BACK_OFF_DELAY_POLICY.delay(0).toMillis()); + + sync = pState.waitForNewMessages(100); + assertThat(sync).completes(ASSERTION_TIMEOUT); + int activeIndex = + consumerStatus.entrySet().stream() + .filter(Map.Entry::getValue) + .map(Map.Entry::getKey) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No active consumer found")); + + sync = consumers.get(activeIndex).waitForNewMessages(100); + assertThat(sync).completes(ASSERTION_TIMEOUT); + + checkConsumers.call(); + + } finally { + if (pState != null) { + pState.close(); + } + consumers.forEach(ConsumerState::close); + if (superStream) { + environment.deleteSuperStream(s); + } else { + environment.deleteStream(s); + } + } + } + private static class ProducerState implements AutoCloseable { + private static final AtomicLong MSG_ID_SEQ = new AtomicLong(0); + private static final byte[] BODY = "hello".getBytes(StandardCharsets.UTF_8); private final String stream; @@ -305,9 +431,19 @@ private static class ProducerState implements AutoCloseable { final AtomicReference lastExceptionInstant = new AtomicReference<>(); private ProducerState(String stream, boolean dynamicBatch, Environment environment) { + this(stream, dynamicBatch, false, environment); + } + + private ProducerState( + String stream, boolean dynamicBatch, boolean superStream, Environment environment) { this.stream = stream; - this.producer = - environment.producerBuilder().stream(stream).dynamicBatch(dynamicBatch).build(); + ProducerBuilder builder = environment.producerBuilder().dynamicBatch(dynamicBatch); + if (superStream) { + builder.superStream(stream).routing(m -> m.getProperties().getMessageIdAsString()); + } else { + builder.stream(stream); + } + this.producer = builder.build(); } void start() { @@ -326,7 +462,14 @@ void start() { try { this.limiter.acquire(1); this.producer.send( - producer.messageBuilder().addData(BODY).build(), confirmationHandler); + producer + .messageBuilder() + .properties() + .messageId(MSG_ID_SEQ.getAndIncrement()) + .messageBuilder() + .addData(BODY) + .build(), + confirmationHandler); } catch (Throwable e) { this.lastException.set(e); this.lastExceptionInstant.set(Instant.now()); @@ -379,16 +522,27 @@ private static class ConsumerState implements AutoCloseable { final AtomicReference postHandle = new AtomicReference<>(() -> {}); private ConsumerState(String stream, Environment environment) { + this(stream, environment, b -> b.stream(stream), (ctx, m) -> {}); + } + + private ConsumerState( + String stream, + Environment environment, + java.util.function.Consumer customizer, + MessageHandler delegateHandler) { this.stream = stream; - this.consumer = - environment.consumerBuilder().stream(stream) + ConsumerBuilder builder = + environment + .consumerBuilder() .offset(OffsetSpecification.first()) .messageHandler( (ctx, m) -> { + delegateHandler.handle(ctx, m); receivedCount.incrementAndGet(); postHandle.get().run(); - }) - .build(); + }); + customizer.accept(builder); + this.consumer = builder.build(); } Sync waitForNewMessages(int messageCount) { @@ -413,4 +567,16 @@ public void close() { this.consumer.close(); } } + + private static void restartCluster() { + nodes.forEach( + n -> { + LOGGER.info("Restarting node {}...", n); + Cli.restartNode(n); + LOGGER.info("Restarted node {}.", n); + }); + LOGGER.info("Rebalancing..."); + Cli.rebalance(); + LOGGER.info("Rebalancing over."); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java index adb6153a27..4faf6cd848 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java @@ -22,7 +22,6 @@ import com.rabbitmq.stream.*; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -66,7 +65,7 @@ public class StompInteroperabilityTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java index dcbb6f158a..583c448e59 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java @@ -15,12 +15,12 @@ package com.rabbitmq.stream.impl; import static com.rabbitmq.stream.ConsumerFlowStrategy.creditWhenHalfMessagesProcessed; +import static com.rabbitmq.stream.impl.Assertions.assertThat; import static com.rabbitmq.stream.impl.TestUtils.*; import static com.rabbitmq.stream.impl.TestUtils.CountDownLatchConditions.completed; -import static java.lang.Runtime.getRuntime; import static java.lang.String.format; import static java.util.Collections.synchronizedList; -import static org.assertj.core.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.rabbitmq.stream.*; import com.rabbitmq.stream.impl.Client.QueryOffsetResponse; @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.IntConsumer; import java.util.function.IntFunction; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -52,10 +53,14 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) public class StreamConsumerTest { + private static final Logger LOGGER = LoggerFactory.getLogger(StreamConsumerTest.class); + static final Duration RECOVERY_DELAY = Duration.ofSeconds(2); static final Duration TOPOLOGY_DELAY = Duration.ofSeconds(2); static volatile Duration recoveryInitialDelay; @@ -149,8 +154,8 @@ void committedOffsetShouldBeSet() throws Exception { }) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); - assertThat(committedOffset.get()).isNotZero(); + org.assertj.core.api.Assertions.assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(committedOffset.get()).isNotZero(); consumer.close(); } @@ -173,7 +178,7 @@ void consume() throws Exception { Collections.singletonList( client.messageBuilder().addData("".getBytes()).build()))); - assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); CountDownLatch consumeLatch = new CountDownLatch(messageCount); @@ -188,8 +193,8 @@ void consume() throws Exception { }) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); - assertThat(chunkTimestamp.get()).isNotZero(); + org.assertj.core.api.Assertions.assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(chunkTimestamp.get()).isNotZero(); consumer.close(); } @@ -228,7 +233,7 @@ void consumeWithAsyncConsumerFlowControl() throws Exception { waitAtMost(() -> receivedMessageCount.get() >= processingLimit); waitUntilStable(receivedMessageCount::get); - assertThat(receivedMessageCount) + org.assertj.core.api.Assertions.assertThat(receivedMessageCount) .hasValueGreaterThanOrEqualTo(processingLimit) .hasValueLessThan(messageCount); @@ -243,8 +248,7 @@ void consumeWithAsyncConsumerFlowControl() throws Exception { void asynchronousProcessingWithFlowControl() { int messageCount = 100_000; publishAndWaitForConfirms(cf, messageCount, stream); - ExecutorService executorService = - Executors.newFixedThreadPool(getRuntime().availableProcessors()); + ExecutorService executorService = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS); try { CountDownLatch latch = new CountDownLatch(messageCount); environment.consumerBuilder().stream(stream) @@ -260,7 +264,7 @@ void asynchronousProcessingWithFlowControl() { ctx.processed(); })) .build(); - assertThat(latch).is(completed()); + org.assertj.core.api.Assertions.assertThat(latch).is(completed()); } finally { executorService.shutdownNow(); } @@ -284,7 +288,7 @@ void closeOnCondition() throws Exception { Collections.singletonList( client.messageBuilder().addData("".getBytes()).build()))); - assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); int messagesToProcess = 20_000; @@ -306,9 +310,9 @@ void closeOnCondition() throws Exception { }) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); consumer.close(); - assertThat(processedMessages).hasValue(messagesToProcess); + org.assertj.core.api.Assertions.assertThat(processedMessages).hasValue(messagesToProcess); } @Test @@ -341,7 +345,7 @@ void consumerShouldBeClosedWhenStreamGetsDeleted(TestInfo info) throws Exception producer.messageBuilder().addData("".getBytes()).build(), confirmationStatus -> publishLatch.countDown())); - assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); CountDownLatch consumeLatch = new CountDownLatch(messageCount); StreamConsumer consumer = @@ -351,14 +355,14 @@ void consumerShouldBeClosedWhenStreamGetsDeleted(TestInfo info) throws Exception .messageHandler((offset, message) -> consumeLatch.countDown()) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); - assertThat(consumer.isOpen()).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumer.isOpen()).isTrue(); environment.deleteStream(s); TestUtils.waitAtMost(10, () -> !consumer.isOpen()); - assertThat(consumer.isOpen()).isFalse(); + org.assertj.core.api.Assertions.assertThat(consumer.isOpen()).isFalse(); } @Test @@ -398,7 +402,7 @@ void manualTrackingConsumerShouldRestartWhereItLeftOff() throws Exception { messageSending.accept(messageCountFirstWave); - assertThat(latchAssert(latchConfirmFirstWave)).completes(); + org.assertj.core.api.Assertions.assertThat(latchAssert(latchConfirmFirstWave)).completes(); int storeEvery = 100; AtomicInteger consumedMessageCount = new AtomicInteger(); @@ -427,22 +431,24 @@ void manualTrackingConsumerShouldRestartWhereItLeftOff() throws Exception { .build(); ConsumerInfo consumerInfo = MonitoringTestUtils.extract(consumer); - assertThat(consumerInfo.getId()).isGreaterThanOrEqualTo(0); - assertThat(consumerInfo.getStream()).isEqualTo(stream); - assertThat(consumerInfo.getSubscriptionClient()).contains(" -> localhost:5552"); - assertThat(consumerInfo.getTrackingClient()).contains(" -> localhost:5552"); + org.assertj.core.api.Assertions.assertThat(consumerInfo.getId()).isGreaterThanOrEqualTo(0); + org.assertj.core.api.Assertions.assertThat(consumerInfo.getStream()).isEqualTo(stream); + org.assertj.core.api.Assertions.assertThat(consumerInfo.getSubscriptionClient()) + .contains(" -> localhost:5552"); + org.assertj.core.api.Assertions.assertThat(consumerInfo.getTrackingClient()) + .contains(" -> localhost:5552"); consumerReference.set(consumer); waitAtMost(10, () -> consumedMessageCount.get() == messageCountFirstWave); - assertThat(lastStoredOffset.get()).isPositive(); + org.assertj.core.api.Assertions.assertThat(lastStoredOffset.get()).isPositive(); consumer.close(); messageSending.accept(messageCountSecondWave); - assertThat(latchAssert(latchConfirmSecondWave)).completes(); + org.assertj.core.api.Assertions.assertThat(latchAssert(latchConfirmSecondWave)).completes(); AtomicLong firstOffset = new AtomicLong(0); consumer = @@ -467,7 +473,8 @@ void manualTrackingConsumerShouldRestartWhereItLeftOff() throws Exception { // there will be the tracking records after the first wave of messages, // messages offset won't be contiguous, so it's not an exact match - assertThat(firstOffset.get()).isGreaterThanOrEqualTo(lastStoredOffset.get()); + org.assertj.core.api.Assertions.assertThat(firstOffset.get()) + .isGreaterThanOrEqualTo(lastStoredOffset.get()); consumer.close(); } @@ -510,7 +517,7 @@ void consumerShouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesRec Cli.killConnection("rabbitmq-stream-consumer-0"); // no messages should have been received - assertThat(consumedCount.get()).isZero(); + org.assertj.core.api.Assertions.assertThat(consumedCount.get()).isZero(); // starting the second wave, it sends a message every 100 ms AtomicBoolean keepPublishing = new AtomicBoolean(true); @@ -528,7 +535,7 @@ void consumerShouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesRec // the consumer should restart consuming with its initial offset spec, "next" try { latchAssert(consumeLatch).completes(recoveryInitialDelay.multipliedBy(2)); - assertThat(bodies).hasSize(1).contains("second wave"); + org.assertj.core.api.Assertions.assertThat(bodies).hasSize(1).contains("second wave"); } finally { keepPublishing.set(false); } @@ -553,7 +560,7 @@ void consumerShouldKeepConsumingAfterDisruption( producer.messageBuilder().addData("".getBytes()).build(), confirmationStatus -> publishLatch.countDown())); - assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(publishLatch.await(10, TimeUnit.SECONDS)).isTrue(); producer.close(); AtomicInteger receivedMessageCount = new AtomicInteger(0); @@ -571,9 +578,9 @@ void consumerShouldKeepConsumingAfterDisruption( }) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); - assertThat(consumer.isOpen()).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumer.isOpen()).isTrue(); disruption.accept(s); @@ -594,13 +601,14 @@ void consumerShouldKeepConsumingAfterDisruption( producerSecondWave.messageBuilder().addData("".getBytes()).build(), confirmationStatus -> publishLatchSecondWave.countDown())); - assertThat(publishLatchSecondWave.await(10, TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThat(publishLatchSecondWave.await(10, TimeUnit.SECONDS)) + .isTrue(); producerSecondWave.close(); latchAssert(consumeLatchSecondWave).completes(recoveryInitialDelay.plusSeconds(2)); - assertThat(receivedMessageCount.get()) + org.assertj.core.api.Assertions.assertThat(receivedMessageCount.get()) .isBetween(messageCount * 2, messageCount * 2 + 1); // there can be a duplicate - assertThat(consumer.isOpen()).isTrue(); + org.assertj.core.api.Assertions.assertThat(consumer.isOpen()).isTrue(); } finally { if (consumer != null) { @@ -808,7 +816,8 @@ void externalOffsetTrackingWithSubscriptionListener() throws Exception { publish.run(); waitAtMost(5, () -> receivedMessages.get() == messageCount); - assertThat(offsetTracking.get()).isGreaterThanOrEqualTo(messageCount - 1); + org.assertj.core.api.Assertions.assertThat(offsetTracking.get()) + .isGreaterThanOrEqualTo(messageCount - 1); Cli.killConnection("rabbitmq-stream-consumer-0"); waitAtMost( @@ -816,7 +825,8 @@ void externalOffsetTrackingWithSubscriptionListener() throws Exception { publish.run(); waitAtMost(5, () -> receivedMessages.get() == messageCount * 2); - assertThat(offsetTracking.get()).isGreaterThanOrEqualTo(messageCount * 2 - 1); + org.assertj.core.api.Assertions.assertThat(offsetTracking.get()) + .isGreaterThanOrEqualTo(messageCount * 2 - 1); } @Test @@ -868,7 +878,8 @@ void duplicatesWhenResubscribeAfterDisconnectionWithLongFlushInterval() throws E }); // we have duplicates because the last stored value is behind and the re-subscription uses it - assertThat(receivedMessages).hasValueGreaterThan(publishedMessages.get()); + org.assertj.core.api.Assertions.assertThat(receivedMessages) + .hasValueGreaterThan(publishedMessages.get()); } @Test @@ -929,7 +940,8 @@ void useSubscriptionListenerToRestartExactlyWhereDesired() throws Exception { latchAssert(poisonLatch).completes(recoveryInitialDelay.plusSeconds(2)); // no duplicates because the custom offset tracking overrides the stored offset in the // subscription listener - assertThat(receivedMessages).hasValue(publishedMessages.get() + 1); + org.assertj.core.api.Assertions.assertThat(receivedMessages) + .hasValue(publishedMessages.get() + 1); } @Test @@ -993,4 +1005,143 @@ void creationShouldFailWithDetailsWhenUnknownHost() { .isInstanceOfAny(ConnectTimeoutException.class, UnknownHostException.class); } } + + @Test + void resetOffsetTrackingFromEnvironment() { + int messageCount = 100; + publishAndWaitForConfirms(cf, messageCount, stream); + String reference = "app"; + Sync sync = sync(messageCount); + AtomicLong lastOffset = new AtomicLong(0); + Supplier consumerSupplier = + () -> + environment.consumerBuilder().stream(stream) + .name(reference) + .offset(OffsetSpecification.first()) + .messageHandler( + (context, message) -> { + lastOffset.set(context.offset()); + sync.down(); + }) + .autoTrackingStrategy() + .builder() + .build(); + // consumer gets the initial message batch and stores the offset on closing + Consumer consumer = consumerSupplier.get(); + assertThat(sync).completes(); + consumer.close(); + + // we'll publish 1 more message and make sure the consumers only consumes that one + // (because it restarts where it left off) + long limit = lastOffset.get(); + sync.reset(1); + consumer = consumerSupplier.get(); + + publishAndWaitForConfirms(cf, 1, stream); + + assertThat(sync).completes(); + org.assertj.core.api.Assertions.assertThat(lastOffset).hasValueGreaterThan(limit); + consumer.close(); + + // we reset the offset to 0, the consumer should restart from the beginning + environment.storeOffset(reference, stream, 0); + sync.reset(messageCount + 1); + consumer = consumerSupplier.get(); + + assertThat(sync).completes(); + consumer.close(); + } + + @Test + void asynchronousProcessingWithInMemoryQueue(TestInfo info) { + int messageCount = 100_000; + publishAndWaitForConfirms(cf, messageCount, stream); + + CountDownLatch latch = new CountDownLatch(messageCount); + + MessageHandler handler = (ctx, msg) -> latch.countDown(); + DispatchingMessageHandler dispatchingHandler = + new DispatchingMessageHandler( + handler, ThreadUtils.threadFactory(info.getTestMethod().get().getName())); + + try { + environment.consumerBuilder().stream(stream) + .offset(OffsetSpecification.first()) + .flow() + .strategy(creditWhenHalfMessagesProcessed(1)) + .builder() + .messageHandler(dispatchingHandler) + .build(); + org.assertj.core.api.Assertions.assertThat(latch).is(completed()); + } finally { + dispatchingHandler.close(); + } + } + + private static final class DispatchingMessageHandler implements MessageHandler, AutoCloseable { + + private final MessageHandler delegate; + private final BlockingQueue queue = new ArrayBlockingQueue<>(10_000); + private final Thread t; + + private DispatchingMessageHandler(MessageHandler delegate, ThreadFactory tf) { + this.delegate = delegate; + t = + tf.newThread( + () -> { + try { + while (!Thread.currentThread().isInterrupted()) { + ContextMessageWrapper item = queue.poll(10, TimeUnit.SECONDS); + if (item != null) { + try { + this.delegate.handle(item.ctx, item.msg()); + } finally { + item.ctx.processed(); + } + } + } + } catch (InterruptedException e) { + // finish the thread + } + }); + t.start(); + } + + @Override + public void handle(Context context, Message message) { + try { + boolean inserted = + this.queue.offer(new ContextMessageWrapper(context, message), 10, TimeUnit.SECONDS); + if (!inserted) { + LOGGER.warn("Failed to insert message into queue"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void close() { + this.t.interrupt(); + } + } + + private static final class ContextMessageWrapper { + + private final MessageHandler.Context ctx; + private final Message msg; + + private ContextMessageWrapper(MessageHandler.Context ctx, Message msg) { + this.ctx = ctx; + this.msg = msg; + } + + private MessageHandler.Context ctx() { + return this.ctx; + } + + private Message msg() { + return this.msg; + } + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java index e080275f1f..f24cccd473 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java @@ -14,6 +14,7 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import static com.rabbitmq.stream.impl.Assertions.assertThat; import static com.rabbitmq.stream.impl.TestUtils.*; import static com.rabbitmq.stream.impl.TestUtils.CountDownLatchConditions.completed; import static com.rabbitmq.stream.impl.TestUtils.ExceptionConditions.responseCode; @@ -51,12 +52,19 @@ import com.rabbitmq.stream.impl.TestUtils.DisabledIfTlsNotEnabled; import io.netty.channel.Channel; import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.IoHandlerFactory; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.epoll.EpollIoHandler; import io.netty.channel.epoll.EpollSocketChannel; +import io.netty.channel.kqueue.KQueueIoHandler; +import io.netty.channel.kqueue.KQueueSocketChannel; +import io.netty.channel.uring.IoUringIoHandler; +import io.netty.channel.uring.IoUringSocketChannel; import io.netty.handler.ssl.SslHandler; import java.net.ConnectException; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -80,11 +88,10 @@ import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -@ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) +@StreamTestInfrastructure public class StreamEnvironmentTest { EnvironmentBuilder environmentBuilder; @@ -121,10 +128,14 @@ void environmentCreationShouldFailWithIncorrectVirtualHostInUri() { } @Test - void environmentCreationShouldFailWithUrlUsingWrongPort() { + void environmentCreationShouldFailWithUrlUsingWrongPort() throws Exception { + Collection initialThreads = threads(); assertThatThrownBy( () -> environmentBuilder + .netty() + .eventLoopGroup(null) + .environmentBuilder() .uri("rabbitmq-stream://guest:guest@localhost:4242/%2f") .addressResolver(address -> new Address("localhost", 4242)) .build() @@ -132,6 +143,21 @@ void environmentCreationShouldFailWithUrlUsingWrongPort() { .isInstanceOf(StreamException.class) .hasCauseInstanceOf(ConnectException.class) .hasRootCauseMessage("Connection refused"); + // no thread leak + waitAtMost( + Duration.ofSeconds(20), + () -> threads().size() <= initialThreads.size(), + () -> { + Collection current = threads(); + Collection diff = Collections.emptySet(); + if (current.size() > initialThreads.size()) { + diff = new ArrayList<>(current); + diff.removeAll(initialThreads); + } + return String.format( + "There should be no new threads, initial %d, current %d (diff: %s)", + initialThreads.size(), current.size(), diff); + }); } @Test @@ -215,7 +241,7 @@ void producersAndConsumersShouldBeClosedWhenEnvironmentIsClosed(boolean lazyInit } @Test - void growShrinkResourcesWhenProducersConsumersAreOpenedAndClosed(TestInfo info) throws Exception { + void growShrinkResourcesWhenProducersConsumersAreOpenedAndClosed(TestInfo info) { int messageCount = 100; int streamCount = 20; int producersCount = ProducersCoordinator.MAX_PRODUCERS_PER_CLIENT * 3 + 10; @@ -722,40 +748,21 @@ void nettyInitializersAreCalled() { @EnabledOnOs(OS.LINUX) @EnabledIfSystemProperty(named = "os.arch", matches = "amd64") void nativeEpollWorksOnLinux() { - int messageCount = 10_000; - EventLoopGroup epollEventLoopGroup = new EpollEventLoopGroup(); - try { - Set channels = ConcurrentHashMap.newKeySet(); - try (Environment env = - environmentBuilder - .netty() - .eventLoopGroup(epollEventLoopGroup) - .bootstrapCustomizer(b -> b.channel(EpollSocketChannel.class)) - .channelCustomizer(ch -> channels.add(ch)) - .environmentBuilder() - .build()) { - Producer producer = env.producerBuilder().stream(this.stream).build(); - ConfirmationHandler handler = confirmationStatus -> {}; - IntStream.range(0, messageCount) - .forEach( - i -> - producer.send( - producer - .messageBuilder() - .addData("hello".getBytes(StandardCharsets.UTF_8)) - .build(), - handler)); - CountDownLatch latch = new CountDownLatch(messageCount); - env.consumerBuilder().stream(this.stream) - .offset(OffsetSpecification.first()) - .messageHandler((context, message) -> latch.countDown()) - .build(); - assertThat(latchAssert(latch)).completes(); - } - assertThat(channels).isNotEmpty().allMatch(ch -> ch instanceof EpollSocketChannel); - } finally { - epollEventLoopGroup.shutdownGracefully(0, 0, SECONDS); - } + nativeIo(EpollIoHandler.newFactory(), EpollSocketChannel.class); + } + + @Test + @EnabledOnOs(OS.LINUX) + @EnabledIfSystemProperty(named = "os.arch", matches = "amd64") + void nativeIoUringWorksOnLinux() { + nativeIo(IoUringIoHandler.newFactory(), IoUringSocketChannel.class); + } + + @Test + @EnabledOnOs(OS.MAC) + @EnabledIfSystemProperty(named = "os.arch", matches = "aarch64") + void nativeKqueueWorksOnMac() { + nativeIo(KQueueIoHandler.newFactory(), KQueueSocketChannel.class); } @Test @@ -810,4 +817,56 @@ void brokerShouldAcceptInitialMemberCountArgument(TestInfo info) { env.close(); } } + + @Test + void storeOffset() { + String ref = "app"; + Client client = cf.get(); + assertThat(client.queryOffset(ref, stream)).isNotOk().hasCodeNoOffset(); + try (Environment env = environmentBuilder.build()) { + env.storeOffset(ref, stream, 42); + assertThat(client.queryOffset(ref, stream).getOffset()).isEqualTo(42); + env.storeOffset(ref, stream, 43); + assertThat(client.queryOffset(ref, stream).getOffset()).isEqualTo(43); + env.storeOffset(ref, stream, 0); + assertThat(client.queryOffset(ref, stream).getOffset()).isEqualTo(0); + } + } + + private void nativeIo(IoHandlerFactory ioHandlerFactory, Class chClass) { + int messageCount = 10_000; + EventLoopGroup epollEventLoopGroup = new MultiThreadIoEventLoopGroup(ioHandlerFactory); + try { + Set channels = ConcurrentHashMap.newKeySet(); + try (Environment env = + environmentBuilder + .netty() + .eventLoopGroup(epollEventLoopGroup) + .bootstrapCustomizer(b -> b.channel(chClass)) + .channelCustomizer(channels::add) + .environmentBuilder() + .build()) { + Producer producer = env.producerBuilder().stream(this.stream).build(); + ConfirmationHandler handler = confirmationStatus -> {}; + IntStream.range(0, messageCount) + .forEach( + i -> + producer.send( + producer + .messageBuilder() + .addData("hello".getBytes(StandardCharsets.UTF_8)) + .build(), + handler)); + CountDownLatch latch = new CountDownLatch(messageCount); + env.consumerBuilder().stream(this.stream) + .offset(OffsetSpecification.first()) + .messageHandler((context, message) -> latch.countDown()) + .build(); + assertThat(latchAssert(latch)).completes(); + } + assertThat(channels).isNotEmpty().allMatch(ch -> ch.getClass().isAssignableFrom(chClass)); + } finally { + epollEventLoopGroup.shutdownGracefully(0, 0, SECONDS); + } + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java index e8d3f25e34..8dafb175ad 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java @@ -26,7 +26,6 @@ import com.rabbitmq.stream.StreamException; import com.rabbitmq.stream.impl.Client.ClientParameters; import com.rabbitmq.stream.impl.StreamEnvironment.LocatorNotAvailableException; -import io.netty.buffer.ByteBufAllocator; import java.net.URI; import java.time.Duration; import java.util.Arrays; @@ -94,7 +93,7 @@ Client.ClientParameters duplicate() { ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, null, - ByteBufAllocator.DEFAULT, + Utils.byteBufAllocator(), false, type -> "locator-connection", cf, @@ -163,7 +162,7 @@ void shouldTryUrisOnInitializationFailure() throws Exception { ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, null, - ByteBufAllocator.DEFAULT, + Utils.byteBufAllocator(), false, type -> "locator-connection", cf, @@ -195,7 +194,7 @@ void shouldNotOpenConnectionWhenLazyInitIsEnabled( ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, null, - ByteBufAllocator.DEFAULT, + Utils.byteBufAllocator(), lazyInit, type -> "locator-connection", cf, diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java index 63254247ad..47780c26c0 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java @@ -666,4 +666,16 @@ void creationShouldFailWithDetailsWhenUnknownHost() { .isInstanceOfAny(ConnectTimeoutException.class, UnknownHostException.class); } } + + @Test + void messageLargerThanMaxFrameSizeShouldThrowException() { + int messageSize = Client.DEFAULT_MAX_FRAME_SIZE + 1; + Producer producer = environment.producerBuilder().stream(stream).build(); + assertThatThrownBy( + () -> + producer.send( + producer.messageBuilder().addData(new byte[messageSize]).build(), + confirmationStatus -> {})) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java index 1150a90842..51f320a46c 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java @@ -75,7 +75,7 @@ public class StreamProducerUnitTest { void init() { mocks = MockitoAnnotations.openMocks(this); executorService = Executors.newScheduledThreadPool(2); - when(channel.alloc()).thenReturn(ByteBufAllocator.DEFAULT); + when(channel.alloc()).thenReturn(Utils.byteBufAllocator()); when(channel.writeAndFlush(Mockito.any())).thenReturn(channelFuture); when(client.allocateNoCheck(any(ByteBufAllocator.class), anyInt())) .thenAnswer( diff --git a/src/test/java/com/rabbitmq/stream/impl/TestUtils.java b/src/test/java/com/rabbitmq/stream/impl/TestUtils.java index 417842be85..5aa4f82e18 100644 --- a/src/test/java/com/rabbitmq/stream/impl/TestUtils.java +++ b/src/test/java/com/rabbitmq/stream/impl/TestUtils.java @@ -42,7 +42,6 @@ import com.rabbitmq.stream.impl.Client.Response; import com.rabbitmq.stream.impl.Client.StreamMetadata; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import io.vavr.Tuple2; import java.io.IOException; import java.lang.annotation.Documented; @@ -74,7 +73,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ExtensionContext.Store.CloseableResource; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; @@ -627,7 +625,7 @@ static EventLoopGroup eventLoopGroup(ExtensionContext context) { @Override public void beforeAll(ExtensionContext context) { - store(context).put("nettyEventLoopGroup", new NioEventLoopGroup()); + store(context).put("nettyEventLoopGroup", Utils.eventLoopGroup()); } @Override @@ -716,7 +714,6 @@ public void afterAll(ExtensionContext context) { try { eventLoopGroup.shutdownGracefully(0, 0, SECONDS).get(10, SECONDS); } catch (InterruptedException e) { - // happens at the end of the test suite LOGGER.debug("Error while asynchronously closing Netty event loop group", e); Thread.currentThread().interrupt(); } catch (Exception e) { @@ -737,12 +734,13 @@ private static Field field(Class cls, String name) { return field; } - private static class ExecutorServiceCloseableResourceWrapper implements CloseableResource { + private static class ExecutorServiceCloseableResourceWrapper implements AutoCloseable { private final ExecutorService executorService; private ExecutorServiceCloseableResourceWrapper() { - this.executorService = Executors.newCachedThreadPool(); + ThreadFactory tf = ThreadUtils.threadFactory("closing-resource-"); + this.executorService = Executors.newCachedThreadPool(tf); } @Override @@ -1066,7 +1064,8 @@ public enum BrokerVersion { RABBITMQ_3_11_11("3.11.11"), RABBITMQ_3_11_14("3.11.14"), RABBITMQ_3_13_0("3.13.0"), - RABBITMQ_4_0_0("4.0.0"); + RABBITMQ_4_0_0("4.0.0"), + RABBITMQ_4_1_2("4.1.2"); final String value; @@ -1184,4 +1183,8 @@ boolean hasCompleted() { return this.latch.get().getCount() == 0; } } + + static Collection threads() { + return Thread.getAllStackTraces().keySet(); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/TlsTest.java b/src/test/java/com/rabbitmq/stream/impl/TlsTest.java index 3959789cda..fe28508874 100644 --- a/src/test/java/com/rabbitmq/stream/impl/TlsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/TlsTest.java @@ -30,10 +30,9 @@ import com.rabbitmq.stream.impl.TestUtils.DisabledIfAuthMechanismSslNotEnabled; import com.rabbitmq.stream.impl.TestUtils.DisabledIfTlsNotEnabled; import com.rabbitmq.stream.sasl.DefaultSaslConfiguration; -import io.netty.channel.Channel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; import java.io.File; import java.io.FileInputStream; import java.net.InetAddress; @@ -50,27 +49,32 @@ import java.util.Collections; import java.util.UUID; import java.util.concurrent.CountDownLatch; -import java.util.function.Consumer; import java.util.stream.IntStream; import javax.net.ssl.SNIHostName; import javax.net.ssl.SSLException; import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLParameters; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.EnumSource; @DisabledIfTlsNotEnabled @ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) +@ParameterizedClass +@EnumSource(names = {"JDK", "OPENSSL"}) public class TlsTest { + @Parameter SslProvider sslProvider; + String stream; TestUtils.ClientFactory cf; int credit = 10; - static SslContext alwaysTrustSslContext() { + SslContext alwaysTrustSslContext() { try { - return SslContextBuilder.forClient().trustManager(TRUST_EVERYTHING_TRUST_MANAGER).build(); + return builder().trustManager(TRUST_EVERYTHING_TRUST_MANAGER).build(); } catch (SSLException e) { throw new RuntimeException(e); } @@ -192,31 +196,34 @@ void unverifiedConnection() { } @Test - void unverifiedConnectionWithSni() { - Consumer channelCustomizer = - ch -> { - SslHandler sslHandler = ch.pipeline().get(SslHandler.class); - if (sslHandler != null) { - SSLParameters sslParameters = sslHandler.engine().getSSLParameters(); - sslParameters.setServerNames(Collections.singletonList(new SNIHostName("localhost"))); - sslHandler.engine().setSSLParameters(sslParameters); - } - }; - cf.get( - new ClientParameters() - .sslContext(alwaysTrustSslContext()) - .channelCustomizer(channelCustomizer)); + void verifiedConnectionWithCorrectServerCertificate() throws Exception { + // in server certificate SAN + String hostname = "localhost"; + SslContext context = builder().trustManager(caCertificate()).build(); + cf.get(new ClientParameters().host(hostname).sslContext(context)); } @Test - void verifiedConnectionWithCorrectServerCertificate() throws Exception { - SslContext context = SslContextBuilder.forClient().trustManager(caCertificate()).build(); - cf.get(new ClientParameters().sslContext(context)); + void verifiedConnectionWithCorrectServerCertificateWithSni() throws Exception { + // not in server certificate SAN, but setting SNI makes it work + String hostname = "127.0.0.1"; + SslContext context = + builder().trustManager(caCertificate()).serverName(new SNIHostName("localhost")).build(); + cf.get(new ClientParameters().host(hostname).sslContext(context)); + } + + @Test + void verifiedConnectionWithCorrectServerCertificateFailsIfHostnameNotInSan() throws Exception { + // not in server certificate SAN + String hostname = "127.0.0.1"; + SslContext context = builder().trustManager(caCertificate()).build(); + assertThatThrownBy(() -> cf.get(new ClientParameters().host(hostname).sslContext(context))) + .hasCauseInstanceOf(SSLHandshakeException.class); } @Test void verifiedConnectionWithWrongServerCertificate() throws Exception { - SslContext context = SslContextBuilder.forClient().trustManager(clientCertificate()).build(); + SslContext context = builder().trustManager(clientCertificate()).build(); assertThatThrownBy(() -> cf.get(new ClientParameters().sslContext(context))) .isInstanceOf(StreamException.class) .hasCauseInstanceOf(SSLHandshakeException.class); @@ -225,7 +232,7 @@ void verifiedConnectionWithWrongServerCertificate() throws Exception { @Test void verifiedConnectionWithCorrectClientPrivateKey() throws Exception { SslContext context = - SslContextBuilder.forClient() + builder() .trustManager(caCertificate()) .keyManager(clientKey(), clientCertificate()) .build(); @@ -239,10 +246,7 @@ void verifiedConnectionWithCorrectClientPrivateKey() throws Exception { void saslExternalShouldSucceedWithUserForClientCertificate() throws Exception { X509Certificate clientCertificate = clientCertificate(); SslContext context = - SslContextBuilder.forClient() - .trustManager(caCertificate()) - .keyManager(clientKey(), clientCertificate) - .build(); + builder().trustManager(caCertificate()).keyManager(clientKey(), clientCertificate).build(); String username = clientCertificate.getSubjectX500Principal().getName(); Cli.rabbitmqctlIgnoreError(format("delete_user %s", username)); @@ -266,10 +270,7 @@ void saslExternalShouldSucceedWithUserForClientCertificate() throws Exception { void saslExternalShouldFailIfNoUserForClientCertificate() throws Exception { X509Certificate clientCertificate = clientCertificate(); SslContext context = - SslContextBuilder.forClient() - .trustManager(caCertificate()) - .keyManager(clientKey(), clientCertificate) - .build(); + builder().trustManager(caCertificate()).keyManager(clientKey(), clientCertificate).build(); String username = clientCertificate.getSubjectX500Principal().getName(); Cli.rabbitmqctlIgnoreError(format("delete_user %s", username)); @@ -286,7 +287,7 @@ void saslExternalShouldFailIfNoUserForClientCertificate() throws Exception { @Test void hostnameVerificationShouldFailWhenSettingHostToLoopbackInterface() throws Exception { - SslContext context = SslContextBuilder.forClient().trustManager(caCertificate()).build(); + SslContext context = builder().trustManager(caCertificate()).build(); assertThatThrownBy(() -> cf.get(new ClientParameters().sslContext(context).host("127.0.0.1"))) .isInstanceOf(StreamException.class) .hasCauseInstanceOf(SSLHandshakeException.class); @@ -295,12 +296,9 @@ void hostnameVerificationShouldFailWhenSettingHostToLoopbackInterface() throws E @Test void shouldConnectWhenSettingHostToLoopbackInterfaceAndDisablingHostnameVerification() throws Exception { - SslContext context = SslContextBuilder.forClient().trustManager(caCertificate()).build(); - cf.get( - new ClientParameters() - .sslContext(context) - .host("127.0.0.1") - .tlsHostnameVerification(false)); + SslContext context = + builder().endpointIdentificationAlgorithm(null).trustManager(caCertificate()).build(); + cf.get(new ClientParameters().sslContext(context).host("127.0.0.1")); } @Test @@ -323,7 +321,7 @@ void environmentPublisherConsumer() throws Exception { .uri("rabbitmq-stream+tls://localhost") .addressResolver(addr -> new Address("localhost", Client.DEFAULT_TLS_PORT)) .tls() - .sslContext(SslContextBuilder.forClient().trustManager(caCertificate()).build()) + .sslContext(builder().trustManager(caCertificate()).build()) .environmentBuilder() .build()) { @@ -369,4 +367,8 @@ private static String hostname() { private static String tlsArtefactPath(String in) { return in.replace("$(hostname)", hostname()).replace("$(hostname -s)", hostname()); } + + private SslContextBuilder builder() { + return SslContextBuilder.forClient().sslProvider(sslProvider); + } } diff --git a/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 0000000000..4b1d329662 --- /dev/null +++ b/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1 @@ +com.rabbitmq.stream.StreamClientTestExtension \ No newline at end of file diff --git a/src/test/resources/junit-platform.properties b/src/test/resources/junit-platform.properties new file mode 100644 index 0000000000..5301ead855 --- /dev/null +++ b/src/test/resources/junit-platform.properties @@ -0,0 +1 @@ +junit.jupiter.extensions.autodetection.enabled=false \ No newline at end of file