Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.appender;
018
019import java.io.File;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.RandomAccessFile;
023import java.io.Serializable;
024import java.lang.reflect.Method;
025import java.nio.ByteBuffer;
026import java.nio.ByteOrder;
027import java.nio.MappedByteBuffer;
028import java.nio.channels.FileChannel;
029import java.security.AccessController;
030import java.security.PrivilegedActionException;
031import java.security.PrivilegedExceptionAction;
032import java.util.HashMap;
033import java.util.Map;
034import java.util.Objects;
035
036import org.apache.logging.log4j.core.Layout;
037import org.apache.logging.log4j.core.util.Closer;
038import org.apache.logging.log4j.core.util.FileUtils;
039import org.apache.logging.log4j.core.util.NullOutputStream;
040import org.apache.logging.log4j.util.Constants;
041
042//Lines too long...
043//CHECKSTYLE:OFF
044/**
045 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
046 * memory and writes to this memory region.
047 * <p>
048 *
049 * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
050 *      http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
051 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
052 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
053 * @see <a
054 *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
055 *      http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
056 *
057 * @since 2.1
058 */
059//CHECKSTYLE:ON
060public class MemoryMappedFileManager extends OutputStreamManager {
061    /**
062     * Default length of region to map.
063     */
064    static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
065    private static final int MAX_REMAP_COUNT = 10;
066    private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
067    private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
068
069    private final boolean immediateFlush;
070    private final int regionLength;
071    private final String advertiseURI;
072    private final RandomAccessFile randomAccessFile;
073    private MappedByteBuffer mappedBuffer;
074    private long mappingOffset;
075
076    protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
077            final boolean immediateFlush, final long position, final int regionLength, final String advertiseURI,
078            final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
079        super(os, fileName, layout, writeHeader, ByteBuffer.wrap(Constants.EMPTY_BYTE_ARRAY));
080        this.immediateFlush = immediateFlush;
081        this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
082        this.regionLength = regionLength;
083        this.advertiseURI = advertiseURI;
084        this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
085        this.byteBuffer = mappedBuffer;
086        this.mappingOffset = position;
087    }
088
089    /**
090     * Returns the MemoryMappedFileManager.
091     *
092     * @param fileName The name of the file to manage.
093     * @param append true if the file should be appended to, false if it should be overwritten.
094     * @param immediateFlush true if the contents should be flushed to disk on every write
095     * @param regionLength The mapped region length.
096     * @param advertiseURI the URI to use when advertising the file
097     * @param layout The layout.
098     * @return A MemoryMappedFileManager for the File.
099     */
100    public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
101            final boolean immediateFlush, final int regionLength, final String advertiseURI,
102            final Layout<? extends Serializable> layout) {
103        return narrow(MemoryMappedFileManager.class, getManager(fileName, new FactoryData(append, immediateFlush,
104                regionLength, advertiseURI, layout), FACTORY));
105    }
106
107    /**
108     * No longer used, the {@link org.apache.logging.log4j.core.LogEvent#isEndOfBatch()} attribute is used instead.
109     * @return {@link Boolean#FALSE}.
110     * @deprecated end-of-batch on the event is used instead.
111     */
112    @Deprecated
113    public Boolean isEndOfBatch() {
114        return Boolean.FALSE;
115    }
116
117    /**
118     * No longer used, the {@link org.apache.logging.log4j.core.LogEvent#isEndOfBatch()} attribute is used instead.
119     * This method is a no-op.
120     * @deprecated end-of-batch on the event is used instead.
121     */
122    @Deprecated
123    public void setEndOfBatch(@SuppressWarnings("unused") final boolean endOfBatch) {
124    }
125
126    @Override
127    protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
128        while (length > mappedBuffer.remaining()) {
129            final int chunk = mappedBuffer.remaining();
130            mappedBuffer.put(bytes, offset, chunk);
131            offset += chunk;
132            length -= chunk;
133            remap();
134        }
135        mappedBuffer.put(bytes, offset, length);
136
137        // no need to call flush() if force is true,
138        // already done in AbstractOutputStreamAppender.append
139    }
140
141    private synchronized void remap() {
142        final long offset = this.mappingOffset + mappedBuffer.position();
143        final int length = mappedBuffer.remaining() + regionLength;
144        try {
145            unsafeUnmap(mappedBuffer);
146            final long fileLength = randomAccessFile.length() + regionLength;
147            LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
148                    regionLength, fileLength);
149
150            final long startNanos = System.nanoTime();
151            randomAccessFile.setLength(fileLength);
152            final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
153            LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
154                    millis);
155
156            mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
157            this.byteBuffer = mappedBuffer;
158            mappingOffset = offset;
159        } catch (final Exception ex) {
160            logError("Unable to remap", ex);
161        }
162    }
163
164    @Override
165    public synchronized void flush() {
166        mappedBuffer.force();
167    }
168
169    @Override
170    public synchronized boolean closeOutputStream() {
171        final long position = mappedBuffer.position();
172        final long length = mappingOffset + position;
173        try {
174            unsafeUnmap(mappedBuffer);
175        } catch (final Exception ex) {
176            logError("Unable to unmap MappedBuffer", ex);
177        }
178        try {
179            LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
180                    length, mappingOffset, position);
181            randomAccessFile.setLength(length);
182            randomAccessFile.close();
183            return true;
184        } catch (final IOException ex) {
185            logError("Unable to close MemoryMappedFile", ex);
186            return false;
187        }
188    }
189
190    public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
191            final int size) throws IOException {
192        for (int i = 1;; i++) {
193            try {
194                LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
195
196                final long startNanos = System.nanoTime();
197                final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
198                map.order(ByteOrder.nativeOrder());
199
200                final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
201                LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
202
203                return map;
204            } catch (final IOException e) {
205                if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
206                    throw e;
207                }
208                LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
209                if (i < MAX_REMAP_COUNT) {
210                    Thread.yield();
211                } else {
212                    try {
213                        Thread.sleep(1);
214                    } catch (final InterruptedException ignored) {
215                        Thread.currentThread().interrupt();
216                        throw e;
217                    }
218                }
219            }
220        }
221    }
222
223    private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
224        LOGGER.debug("MMapAppender unmapping old buffer...");
225        final long startNanos = System.nanoTime();
226        AccessController.doPrivileged((PrivilegedExceptionAction<Object>) () -> {
227            final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
228            getCleanerMethod.setAccessible(true);
229            final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
230            final Method cleanMethod = cleaner.getClass().getMethod("clean");
231            cleanMethod.invoke(cleaner);
232            return null;
233        });
234        final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
235        LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
236    }
237
238    /**
239     * Returns the name of the File being managed.
240     *
241     * @return The name of the File being managed.
242     */
243    public String getFileName() {
244        return getName();
245    }
246
247    /**
248     * Returns the length of the memory mapped region.
249     *
250     * @return the length of the mapped region
251     */
252    public int getRegionLength() {
253        return regionLength;
254    }
255
256    /**
257     * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
258     * {@code false} otherwise.
259     *
260     * @return whether each write should be force-sync'ed
261     */
262    public boolean isImmediateFlush() {
263        return immediateFlush;
264    }
265
266    /**
267     * Gets this FileManager's content format specified by:
268     * <p>
269     * Key: "fileURI" Value: provided "advertiseURI" param.
270     * </p>
271     *
272     * @return Map of content format keys supporting FileManager
273     */
274    @Override
275    public Map<String, String> getContentFormat() {
276        final Map<String, String> result = new HashMap<>(super.getContentFormat());
277        result.put("fileURI", advertiseURI);
278        return result;
279    }
280
281    @Override
282    protected void flushBuffer(final ByteBuffer buffer) {
283        // do nothing (do not call drain() to avoid spurious remapping)
284    }
285
286    @Override
287    public ByteBuffer getByteBuffer() {
288        return mappedBuffer;
289    }
290
291    @Override
292    public ByteBuffer drain(final ByteBuffer buf) {
293        remap();
294        return mappedBuffer;
295    }
296
297    /**
298     * Factory Data.
299     */
300    private static class FactoryData {
301        private final boolean append;
302        private final boolean immediateFlush;
303        private final int regionLength;
304        private final String advertiseURI;
305        private final Layout<? extends Serializable> layout;
306
307        /**
308         * Constructor.
309         *
310         * @param append Append to existing file or truncate.
311         * @param immediateFlush forces the memory content to be written to the storage device on every event
312         * @param regionLength length of the mapped region
313         * @param advertiseURI the URI to use when advertising the file
314         * @param layout The layout.
315         */
316        public FactoryData(final boolean append, final boolean immediateFlush, final int regionLength,
317                final String advertiseURI, final Layout<? extends Serializable> layout) {
318            this.append = append;
319            this.immediateFlush = immediateFlush;
320            this.regionLength = regionLength;
321            this.advertiseURI = advertiseURI;
322            this.layout = layout;
323        }
324    }
325
326    /**
327     * Factory to create a MemoryMappedFileManager.
328     */
329    private static class MemoryMappedFileManagerFactory
330            implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
331
332        /**
333         * Create a MemoryMappedFileManager.
334         *
335         * @param name The name of the File.
336         * @param data The FactoryData
337         * @return The MemoryMappedFileManager for the File.
338         */
339        @SuppressWarnings("resource")
340        @Override
341        public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
342            final File file = new File(name);
343            if (!data.append) {
344                file.delete();
345            }
346
347            final boolean writeHeader = !data.append || !file.exists();
348            final OutputStream os = NullOutputStream.getInstance();
349            RandomAccessFile raf = null;
350            try {
351                FileUtils.makeParentDirs(file);
352                raf = new RandomAccessFile(name, "rw");
353                final long position = (data.append) ? raf.length() : 0;
354                raf.setLength(position + data.regionLength);
355                return new MemoryMappedFileManager(raf, name, os, data.immediateFlush, position, data.regionLength,
356                        data.advertiseURI, data.layout, writeHeader);
357            } catch (final Exception ex) {
358                LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
359                Closer.closeSilently(raf);
360            }
361            return null;
362        }
363    }
364}