This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch GLUTEN-11658-restore-scala-recompile-mode in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
commit f6dbcbe35785efa274c0a9f89cf17497b14535e0 Author: Chang chen <[email protected]> AuthorDate: Thu Feb 26 10:20:15 2026 +0000 [GLUTEN-11658][CORE] Restore scala.recompile.mode default to 'all' and introduce fast-build profile - Restore scala.recompile.mode default from 'incremental' back to 'all' to fix CI/clean build failures caused by Zinc's incremental compiler not tracking transitive Spark dependencies correctly - Rename the 'bloop' Maven profile (added in #11645) to 'fast-build' and add scala.recompile.mode=incremental inside it; used by bloop-setup.sh and run-scala-test.sh where Zinc analysis is warm and reliable - Simplify run-scala-test.sh: replace 5 redundant -D skip flags with -Pfast-build profile activation - Add pathing JAR support in run-scala-test.sh to avoid OS command-line length limits when classpath is very long Closes #11658 Co-Authored-By: Claude Sonnet 4.6 <[email protected]> --- dev/bloop-setup.sh | 2 +- dev/run-scala-test.sh | 106 +++++++++++++++++++++++++---------- docs/developers/bloop-integration.md | 2 +- pom.xml | 8 ++- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/dev/bloop-setup.sh b/dev/bloop-setup.sh index e633e2453b..8a2eb0b78c 100755 --- a/dev/bloop-setup.sh +++ b/dev/bloop-setup.sh @@ -190,7 +190,7 @@ fi # Skip style checks for faster generation log_step "Running generate-sources + bloopInstall..." ./build/mvn generate-sources bloop:bloopInstall \ - -P"${PROFILES}",bloop \ + -P"${PROFILES}",fast-build \ -DskipTests if [[ ! -d ".bloop" ]]; then diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index 6f3f8688d3..cfe195f939 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -527,14 +527,9 @@ fi ${MVN_CMD} ${MVN_GOALS} \ -T 1C -q \ -pl "${MODULE}" -am \ - -P"${PROFILES}" \ + -P"${PROFILES},fast-build" \ -DincludeScope=test \ -Dmdep.outputFile="${CLASSPATH_FILE}" \ - -Dspotless.check.skip=true \ - -Dscalastyle.skip=true \ - -Dcheckstyle.skip=true \ - -Dmaven.gitcommitid.skip=true \ - -Dremoteresources.skip=true \ ${EXTRA_MVN_ARGS} if [[ ! -f "${CLASSPATH_FILE}" ]]; then @@ -617,14 +612,82 @@ TIMING_STEP3=$(timer_elapsed $TIMER_STEP3_START $TIMER_STEP3_END) log_timing "Step 3 - Resolve classpath" "$TIMING_STEP3" # ============================================================================= -# Step 3.5: Export-only mode (if requested) +# Step 3.5: Create pathing JAR +# ============================================================================= +# A "pathing JAR" is a thin JAR whose MANIFEST.MF Class-Path header lists all +# the real classpath entries as file: URIs. This avoids exceeding OS command-line +# length limits. Works on Java 8+ (unlike @argfile which requires Java 9+). +# Also includes META-INF/classpath.txt for human-readable review. +# ============================================================================= + +PATHING_JAR="/tmp/gluten-test-pathing-$$.jar" +rm -f "${PATHING_JAR}" + +PATHING_MANIFEST_DIR="/tmp/gluten-test-manifest-$$" +mkdir -p "${PATHING_MANIFEST_DIR}/META-INF" + +# Convert colon-separated classpath to space-separated file: URIs for manifest. +# Also write a human-readable classpath listing (one entry per line, same order). +CP_URIS="" +IFS=':' read -ra _CP_ITEMS <<< "${RESOLVED_CLASSPATH}" +: > "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + +for _item in "${_CP_ITEMS[@]}"; do + [[ -z "$_item" ]] && continue + echo "$_item" >> "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt" + # Ensure directories end with / (required by Class-Path spec for directories) + [[ -d "$_item" && "$_item" != */ ]] && _item="${_item}/" + CP_URIS="${CP_URIS} file://${_item}" +done +CP_URIS="${CP_URIS# }" + +# Write manifest with proper 72-byte line wrapping. +# MANIFEST.MF spec: max 72 bytes per line; continuation lines start with +# a single space character. +{ + echo "Manifest-Version: 1.0" + _mf_header="Class-Path: ${CP_URIS}" + echo "${_mf_header:0:72}" + _mf_rest="${_mf_header:72}" + while [[ -n "$_mf_rest" ]]; do + echo " ${_mf_rest:0:71}" + _mf_rest="${_mf_rest:71}" + done + echo "" +} > "${PATHING_MANIFEST_DIR}/META-INF/MANIFEST.MF" + +# Build the pathing JAR (manifest + human-readable classpath listing) +(cd "${PATHING_MANIFEST_DIR}" && jar cfm "${PATHING_JAR}" META-INF/MANIFEST.MF META-INF/classpath.txt) +rm -rf "${PATHING_MANIFEST_DIR}" +log_info "Created pathing JAR: ${PATHING_JAR} ($(du -h "${PATHING_JAR}" | cut -f1))" +log_info " Review classpath: unzip -p ${PATHING_JAR} META-INF/classpath.txt" + +# ============================================================================= +# Build java command (shared by export-only and run modes) +# ============================================================================= + +JAVA_CMD="${JAVA_HOME}/bin/java" +[[ ! -x "$JAVA_CMD" ]] && JAVA_CMD="java" + +JAVA_ARGS=( + ${JVM_ARGS} + "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties" + -cp "${PATHING_JAR}" + org.scalatest.tools.Runner + -oDF + -s "${SUITE}" +) +[[ -n "$TEST_METHOD" ]] && JAVA_ARGS+=(-t "${TEST_METHOD}") + +# ============================================================================= +# Step 3.6: Export-only mode (if requested) # ============================================================================= if [[ "$EXPORT_ONLY" == "true" ]]; then - EXPORT_FILE="/tmp/gluten-classpath-exported.txt" - echo "$RESOLVED_CLASSPATH" > "$EXPORT_FILE" - log_info "✓ Classpath exported to: ${EXPORT_FILE}" - log_info " Use with: java <jvm-args> -cp \"\$(cat ${EXPORT_FILE})\" org.scalatest.tools.Runner -s <suite>" + echo "" + echo -e "${YELLOW}# Run the test with:${NC}" + echo "${JAVA_CMD} ${JAVA_ARGS[*]}" + echo "" print_timing_summary false exit 0 fi @@ -635,12 +698,6 @@ fi log_step "Step 4: Running ScalaTest..." -# Find Java -JAVA_CMD="${JAVA_HOME}/bin/java" -if [[ ! -x "$JAVA_CMD" ]]; then - JAVA_CMD="java" -fi - log_info "Suite: ${SUITE}" [[ -n "$TEST_METHOD" ]] && log_info "Test method: ${TEST_METHOD}" @@ -650,21 +707,12 @@ echo "Running ScalaTest" echo "==========================================" echo "" -# Build test method args conditionally -TEST_METHOD_ARGS=() -[[ -n "$TEST_METHOD" ]] && TEST_METHOD_ARGS=(-t "${TEST_METHOD}") - TIMER_STEP4_START=$(timer_now) TEST_EXIT_CODE=0 -${JAVA_CMD} \ - ${JVM_ARGS} \ - -Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties \ - -cp "${RESOLVED_CLASSPATH}" \ - org.scalatest.tools.Runner \ - -s "${SUITE}" \ - "${TEST_METHOD_ARGS[@]}" \ - -oDF || TEST_EXIT_CODE=$? +${JAVA_CMD} "${JAVA_ARGS[@]}" || TEST_EXIT_CODE=$? + +rm -f "${PATHING_JAR}" TIMER_STEP4_END=$(timer_now) TIMING_STEP4=$(timer_elapsed $TIMER_STEP4_START $TIMER_STEP4_END) diff --git a/docs/developers/bloop-integration.md b/docs/developers/bloop-integration.md index 2364b8f596..c5efdbbd52 100644 --- a/docs/developers/bloop-integration.md +++ b/docs/developers/bloop-integration.md @@ -65,7 +65,7 @@ The `-Pbloop` profile automatically skips style checks during configuration gene ./dev/bloop-setup.sh -Pspark-3.5,scala-2.12,backends-velox # Manual invocation with profile -./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,bloop -DskipTests +./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,fast-build -DskipTests ``` The bloop profile sets these properties automatically: diff --git a/pom.xml b/pom.xml index 9af9927f9e..0f48bb5827 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ --> <os.full.name>unknown</os.full.name> <!-- To build built-in backend c++ codes --> - <scala.recompile.mode>incremental</scala.recompile.mode> + <scala.recompile.mode>all</scala.recompile.mode> <!-- For unit tests --> <fasterxml.version>2.15.0</fasterxml.version> <junit.version>4.13.1</junit.version> @@ -2147,17 +2147,19 @@ </build> </profile> <profile> - <id>bloop</id> + <id>fast-build</id> <activation> <activeByDefault>false</activeByDefault> </activation> <properties> - <!-- Skip style checks during bloop configuration generation --> + <!-- Skip style checks for faster local builds --> <spotless.check.skip>true</spotless.check.skip> <scalastyle.skip>true</scalastyle.skip> <checkstyle.skip>true</checkstyle.skip> <maven.gitcommitid.skip>true</maven.gitcommitid.skip> <remoteresources.skip>true</remoteresources.skip> + <!-- Use incremental Scala compilation (Zinc state is warm in local/dev contexts) --> + <scala.recompile.mode>incremental</scala.recompile.mode> </properties> </profile> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
