davsclaus commented on code in PR #21249:
URL: https://github.com/apache/camel/pull/21249#discussion_r2767772816
##########
components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipSplitterRouteTest.java:
##########
@@ -16,36 +16,174 @@
*/
package org.apache.camel.dataformat.zipfile;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.spi.StreamCachingStrategy;
import org.apache.camel.test.junit5.CamelTestSupport;
+import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
public class ZipSplitterRouteTest extends CamelTestSupport {
+ private static final Logger LOG =
LoggerFactory.getLogger(ZipIterator.class);
+
+ private static final File testDirectory = new File("test/in");
+
+ private final AtomicInteger memoryMbBefore = new AtomicInteger(0);
+ private final AtomicInteger memoryMbInside = new AtomicInteger(0);
@Test
- public void testSplitter() throws InterruptedException {
+ public void testSplitter() throws InterruptedException, IOException {
+ File srcFile = new
File("src/test/resources/org/apache/camel/dataformat/zipfile/data/resources.zip");
+ File testFile = new File(testDirectory, srcFile.getName());
+
+ FileUtils.copyFile(srcFile, testFile);
+
MockEndpoint processZipEntry = getMockEndpoint("mock:processZipEntry");
processZipEntry.expectedBodiesReceivedInAnyOrder("chau", "hi", "hola",
"another_chiau", "another_hi");
MockEndpoint.assertIsSatisfied(context);
}
+ /**
+ * Test that ZipSplitter doesn't read the whole files in the zip file into
memory when Spool is Enabled in the
+ * Stream Caching Strategy
+ */
+ @Test
+ public void testSplitterLargeFileWithSpoolEnabled() throws
InterruptedException, IOException {
+ File testFile = new File(testDirectory, "large1.zip");
+
+ int diff = testSplitterLargeFile(testFile);
+
+ assertThat("Memory spike detected! " + diff + "MB increased.", diff,
lessThan(10));
+ }
+
+ /**
+ * Test that ZipSplitter read the whole files in the zip file into memory
when Spool is Disabled in the Stream
+ * Caching Strategy
+ */
+ @Test
+ public void testSplitterLargeFileWithoutSpoolEnabled() throws
InterruptedException, IOException {
+ File testFile = new File(testDirectory, "large2.zip");
+
+ int diff = testSplitterLargeFile(testFile);
+
+ assertThat("Memory spike detected! " + diff + "MB increased.", diff,
greaterThan(10));
+ }
+
+ private int testSplitterLargeFile(File testFile) throws IOException,
FileNotFoundException, InterruptedException {
+ String expectedBody = null;
+
+ System.out.println("Generating 50MB test file...");
+ try (OutputStream os = new FileOutputStream(testFile);
+ ZipOutputStream zos = new ZipOutputStream(os)) {
+ zos.putNextEntry(new ZipEntry("test.txt"));
+ byte[] chunk = new byte[1024 * 1024];
+ Arrays.fill(chunk, (byte) 'A');
+
+ expectedBody = new String(chunk, 0, 20);
+ for (int i = 1; i <= 50; i++) {
+ zos.write(chunk);
+ }
+ zos.closeEntry();
+ zos.flush();
+ }
+
+ MockEndpoint processZipEntry = getMockEndpoint("mock:processZipEntry");
+ processZipEntry.expectedBodiesReceivedInAnyOrder(expectedBody);
+ MockEndpoint.assertIsSatisfied(context);
+
+ int before = memoryMbBefore.get();
+ int inside = memoryMbInside.get();
+ int diff = inside - before;
+
+ LOG.info("Memory before {}MB, inside {}MB & diff {}MB", before,
inside, diff);
+
+ return diff;
+ }
+
+ @Override
+ protected void setupResources() {
+ if (testDirectory.exists()) {
+ try {
+ FileUtils.deleteDirectory(testDirectory);
+ } catch (IOException e) {
+ LOG.warn("Failed to delete test directory: " + testDirectory,
e);
+ }
+ }
+
+ if (!testDirectory.mkdirs()) {
+ LOG.warn("Failed to create test directory: {}", testDirectory);
+ }
+ }
+
+ @Override
+ protected CamelContext createCamelContext() throws Exception {
+ CamelContext context = super.createCamelContext();
+
+ if
("testSplitterLargeFileWithSpoolEnabled()".equals(contextManagerExtension.getCurrentTestName()))
{
+ StreamCachingStrategy streamCachingStrategy =
context.getStreamCachingStrategy();
+ streamCachingStrategy.setSpoolEnabled(true);
Review Comment:
you can lower the threshold, configure stream caching to use a special
folder for temp files, and during splitting then check that there is a file in
the temp folder.
Some of the existing tests does something like that.
You can then also have another test that is more like an example (disabled
normally) that then create a big file, and then you can use that to test
manually by running the JVM with low memory etc.
Or build a small example for jbang etc
https://github.com/apache/camel-jbang-examples
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
