Chronicle Queue减速并耗尽内存

时间：2017-05-18 21:49:36

标签： java chronicle chronicle-queue

我正在评估Chronicle Queue在我们软件中的使用情况，我必须做错事 我有一个appender，会很快写出约650k条目。之后它停止了，此时内存已经飙升到最大允许值，最终命中OutOfMemory。

这是我的代码：

final class LogEntryOutput implements WriteBytesMarshallable
{
  private final int maxMessageSize;
  private TLogEntry logEntry;

  LogEntryOutput(final int maxMessageSize)
  {
    this.maxMessageSize = maxMessageSize;
  }

  public void setMarshallable(final TLogEntry logEntry)
  {
    this.logEntry = logEntry;
  }

  @Override
  @SuppressWarnings({"rawtypes", "No way to provide generic type and override WriteBytesMarshallable."})
  public void writeMarshallable(final BytesOut bytes)
  {
    bytes.writeLong(this.logEntry.getSessionId());
    bytes.writeInt(this.logEntry.getLogLevel());
    bytes.writeInt(this.logEntry.getSecurityLevel());
    bytes.writeLong(this.logEntry.getPosixTimestamp());

    // Limit size of string messages.
    final int messageSize = Math.min(this.logEntry.getMessage().length(), this.maxMessageSize);

    // Write message length
    bytes.writeStopBit((long)messageSize);

    // Write message bytes.
    bytes.write(this.logEntry.getMessage(), 0, messageSize);
  }
}    

final TLogEntry entry = new TLogEntry();
entry.setSessionId(321234L);
entry.setLogLevel(77);
entry.setSecurityLevel(1234);
entry.setPosixTimestamp(6141234321L);
entry.setMessage("This is a test message for the system................................ A");

final LogEntryOutput output = new LogEntryOutput(1024);
output.setMarshallable(entry);

final ChronicleQueue queue = SingleChronicleQueueBuilder.binary(config.getQueueDirectory())
  .rollCycle(RollCycles.HOURLY)
  .build();
final ExcerptAppender appender = queue.acquireAppender();

for (int j = 0; j < 100; ++j)
{
  for (int i = 0; i < 10000; ++i)
  {
    appender.writeBytes(output);
  }

  System.out.println((j+1) * 10000);
  Jvm.pause(100L);
}

queue.close();

这是在带有64位JVM的Windows 7 x64中运行，使用：-Xmx1024m
我可能做错了什么想法？

编辑：我有其他信息。在内存峰值之后，我拍摄了对象分配的快照。很多对象数组等。 enter image description here enter image description here 当我收到OOM错误时，堆栈跟踪。

java.lang.OutOfMemoryError: GC overhead limit exceeded
    at java.util.concurrent.ConcurrentLinkedQueue.offer(ConcurrentLinkedQueue.java:328)
    at java.util.concurrent.ConcurrentLinkedQueue.add(ConcurrentLinkedQueue.java:297)
    at net.openhft.chronicle.core.ReferenceCounter.recordRelease(ReferenceCounter.java:88)
    at net.openhft.chronicle.core.ReferenceCounter.release(ReferenceCounter.java:79)
    at net.openhft.chronicle.bytes.NativeBytesStore.release(NativeBytesStore.java:267)
    at net.openhft.chronicle.bytes.MappedBytes.acquireNextByteStore(MappedBytes.java:186)
    at net.openhft.chronicle.bytes.MappedBytes.peekVolatileInt(MappedBytes.java:388)
    at net.openhft.chronicle.wire.AbstractWire.readMetaDataHeader(AbstractWire.java:222)
    at net.openhft.chronicle.queue.impl.single.SCQIndexing.arrayForAddress(SCQIndexing.java:190)
    at net.openhft.chronicle.queue.impl.single.SCQIndexing.sequenceForPosition(SCQIndexing.java:492)
    at net.openhft.chronicle.queue.impl.single.SingleChronicleQueueStore.sequenceForPosition(SingleChronicleQueueStore.java:272)
    at net.openhft.chronicle.queue.impl.single.SingleChronicleQueueExcerpts$StoreAppender.checkWritePositionHeaderNumber(SingleChronicleQueueExcerpts.java:339)
    at net.openhft.chronicle.queue.impl.single.SingleChronicleQueueExcerpts$StoreAppender.writingDocument(SingleChronicleQueueExcerpts.java:267)
    at net.openhft.chronicle.wire.MarshallableOut.writingDocument(MarshallableOut.java:55)
    at net.openhft.chronicle.queue.impl.single.SingleChronicleQueueExcerpts$StoreAppender.writeBytes(SingleChronicleQueueExcerpts.java:117)
    at com.selinc.winchester.ledger.writer.harness.queue.LogEntryConsumers$LogEntryChronicle.accept(LogEntryConsumers.java:78)
    at com.selinc.winchester.ledger.writer.harness.queue.LogEntryConsumers$LogEntryChronicle.accept(LogEntryConsumers.java:45)
    at com.selinc.winchester.ledger.writer.harness.queue.LogEntryConsumersTest.test(LogEntryConsumersTest.java:56)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:86)
    at org.testng.internal.Invoker.invokeMethod(Invoker.java:643)
    at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:820)
    at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1128)
    at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:129)
    at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:112)
    at org.testng.TestRunner.privateRun(TestRunner.java:782)
    at org.testng.TestRunner.run(TestRunner.java:632)
    at org.testng.SuiteRunner.runTest(SuiteRunner.java:366)
    at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:361)

1 个答案:

答案 0 :(得分：1)

Chronicle Queue有一些额外的检查来检测使用public class ATest { static class TLogEntry { private long sessionId; private int logLevel; private int securityLevel; private long posixTimestamp; private CharSequence message; public long getSessionId() { return sessionId; } public void setSessionId(long sessionId) { this.sessionId = sessionId; } public int getLogLevel() { return logLevel; } public void setLogLevel(int logLevel) { this.logLevel = logLevel; } public int getSecurityLevel() { return securityLevel; } public void setSecurityLevel(int securityLevel) { this.securityLevel = securityLevel; } public long getPosixTimestamp() { return posixTimestamp; } public void setPosixTimestamp(long posixTimestamp) { this.posixTimestamp = posixTimestamp; } public CharSequence getMessage() { return message; } public void setMessage(CharSequence message) { this.message = message; } } static class LogEntryOutput implements WriteBytesMarshallable { private final int maxMessageSize; private TLogEntry logEntry; LogEntryOutput(final int maxMessageSize) { this.maxMessageSize = maxMessageSize; } public void setMarshallable(final TLogEntry logEntry) { this.logEntry = logEntry; } @Override @SuppressWarnings({"rawtypes", "No way to provide generic type and override WriteBytesMarshallable."}) public void writeMarshallable(final BytesOut bytes) { bytes.writeLong(this.logEntry.getSessionId()); bytes.writeInt(this.logEntry.getLogLevel()); bytes.writeInt(this.logEntry.getSecurityLevel()); bytes.writeLong(this.logEntry.getPosixTimestamp()); // Limit size of string messages. final int messageSize = Math.min(this.logEntry.getMessage().length(), this.maxMessageSize); // Write message length bytes.writeStopBit((long) messageSize); // Write message bytes. bytes.write(this.logEntry.getMessage(), 0, messageSize); } } @Test public void test() { final TLogEntry entry = new TLogEntry(); entry.setSessionId(321234L); entry.setLogLevel(77); entry.setSecurityLevel(1234); entry.setPosixTimestamp(6141234321L); entry.setMessage("This is a test message for the system................................ A"); final LogEntryOutput output = new LogEntryOutput(1024); output.setMarshallable(entry); final ChronicleQueue queue = SingleChronicleQueueBuilder.binary( OS.TARGET + "/test-" + System.nanoTime()) .rollCycle(RollCycles.HOURLY) .build(); final ExcerptAppender appender = queue.acquireAppender(); Jvm.setExceptionHandlers(Slf4jExceptionHandler.FATAL, Slf4jExceptionHandler.WARN, Slf4jExceptionHandler.WARN); for (int j = 0; j < 1000; ++j) { for (int i = 0; i < 10000; ++i) { appender.writeBytes(output); } System.out.println((j + 1) * 10000); // Jvm.pause(100L); } queue.close(); } } 打开的内存泄漏。如果您运行这些额外的检查，队列确实会减慢大约90,000条消息。如果你关闭断言，你会让它运行更长时间。

在具有8 GB内存的Windows笔记本电脑上，在5.5秒内运行10,000,000个条目，关闭断言

它还在66秒内创造了1亿的记录。

HttpClient / HttpWebRequest
相关问题
最新问题