Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.appender;
018
019import java.io.File;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.RandomAccessFile;
023import java.io.Serializable;
024import java.lang.reflect.Method;
025import java.nio.ByteBuffer;
026import java.nio.ByteOrder;
027import java.nio.MappedByteBuffer;
028import java.nio.channels.FileChannel;
029import java.security.AccessController;
030import java.security.PrivilegedActionException;
031import java.security.PrivilegedExceptionAction;
032import java.util.HashMap;
033import java.util.Map;
034import java.util.Objects;
035
036import org.apache.logging.log4j.core.Layout;
037import org.apache.logging.log4j.core.util.Closer;
038import org.apache.logging.log4j.core.util.FileUtils;
039import org.apache.logging.log4j.core.util.NullOutputStream;
040
041//Lines too long...
042//CHECKSTYLE:OFF
043/**
044 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
045 * memory and writes to this memory region.
046 * <p>
047 *
048 * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
049 *      http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
050 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
051 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
052 * @see <a
053 *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
054 *      http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
055 *
056 * @since 2.1
057 */
058//CHECKSTYLE:ON
059public class MemoryMappedFileManager extends OutputStreamManager {
060    /**
061     * Default length of region to map.
062     */
063    static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
064    private static final int MAX_REMAP_COUNT = 10;
065    private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
066    private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
067
068    private final boolean immediateFlush;
069    private final int regionLength;
070    private final String advertiseURI;
071    private final RandomAccessFile randomAccessFile;
072    private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
073    private MappedByteBuffer mappedBuffer;
074    private long mappingOffset;
075
076    protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
077            final boolean immediateFlush, final long position, final int regionLength, final String advertiseURI,
078            final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
079        super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0]));
080        this.immediateFlush = immediateFlush;
081        this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
082        this.regionLength = regionLength;
083        this.advertiseURI = advertiseURI;
084        this.isEndOfBatch.set(Boolean.FALSE);
085        this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
086        this.byteBuffer = mappedBuffer;
087        this.mappingOffset = position;
088    }
089
090    /**
091     * Returns the MemoryMappedFileManager.
092     *
093     * @param fileName The name of the file to manage.
094     * @param append true if the file should be appended to, false if it should be overwritten.
095     * @param immediateFlush true if the contents should be flushed to disk on every write
096     * @param regionLength The mapped region length.
097     * @param advertiseURI the URI to use when advertising the file
098     * @param layout The layout.
099     * @return A MemoryMappedFileManager for the File.
100     */
101    public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
102            final boolean immediateFlush, final int regionLength, final String advertiseURI,
103            final Layout<? extends Serializable> layout) {
104        return narrow(MemoryMappedFileManager.class, getManager(fileName, new FactoryData(append, immediateFlush,
105                regionLength, advertiseURI, layout), FACTORY));
106    }
107
108    public Boolean isEndOfBatch() {
109        return isEndOfBatch.get();
110    }
111
112    public void setEndOfBatch(final boolean endOfBatch) {
113        this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
114    }
115
116    @Override
117    protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
118        while (length > mappedBuffer.remaining()) {
119            final int chunk = mappedBuffer.remaining();
120            mappedBuffer.put(bytes, offset, chunk);
121            offset += chunk;
122            length -= chunk;
123            remap();
124        }
125        mappedBuffer.put(bytes, offset, length);
126
127        // no need to call flush() if force is true,
128        // already done in AbstractOutputStreamAppender.append
129    }
130
131    private synchronized void remap() {
132        final long offset = this.mappingOffset + mappedBuffer.position();
133        final int length = mappedBuffer.remaining() + regionLength;
134        try {
135            unsafeUnmap(mappedBuffer);
136            final long fileLength = randomAccessFile.length() + regionLength;
137            LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
138                    regionLength, fileLength);
139
140            final long startNanos = System.nanoTime();
141            randomAccessFile.setLength(fileLength);
142            final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
143            LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
144                    millis);
145
146            mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
147            this.byteBuffer = mappedBuffer;
148            mappingOffset = offset;
149        } catch (final Exception ex) {
150            logError("Unable to remap", ex);
151        }
152    }
153
154    @Override
155    public synchronized void flush() {
156        mappedBuffer.force();
157    }
158
159    @Override
160    public synchronized boolean closeOutputStream() {
161        final long position = mappedBuffer.position();
162        final long length = mappingOffset + position;
163        try {
164            unsafeUnmap(mappedBuffer);
165        } catch (final Exception ex) {
166            logError("Unable to unmap MappedBuffer", ex);
167        }
168        try {
169            LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
170                    length, mappingOffset, position);
171            randomAccessFile.setLength(length);
172            randomAccessFile.close();
173            return true;
174        } catch (final IOException ex) {
175            logError("Unable to close MemoryMappedFile", ex);
176            return false;
177        }
178    }
179
180    public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
181            final int size) throws IOException {
182        for (int i = 1;; i++) {
183            try {
184                LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
185
186                final long startNanos = System.nanoTime();
187                final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
188                map.order(ByteOrder.nativeOrder());
189
190                final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
191                LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
192
193                return map;
194            } catch (final IOException e) {
195                if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
196                    throw e;
197                }
198                LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
199                if (i < MAX_REMAP_COUNT) {
200                    Thread.yield();
201                } else {
202                    try {
203                        Thread.sleep(1);
204                    } catch (final InterruptedException ignored) {
205                        Thread.currentThread().interrupt();
206                        throw e;
207                    }
208                }
209            }
210        }
211    }
212
213    private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
214        LOGGER.debug("MMapAppender unmapping old buffer...");
215        final long startNanos = System.nanoTime();
216        AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
217            @Override
218            public Object run() throws Exception {
219                final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
220                getCleanerMethod.setAccessible(true);
221                final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
222                final Method cleanMethod = cleaner.getClass().getMethod("clean");
223                cleanMethod.invoke(cleaner);
224                return null;
225            }
226        });
227        final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
228        LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
229    }
230
231    /**
232     * Returns the name of the File being managed.
233     *
234     * @return The name of the File being managed.
235     */
236    public String getFileName() {
237        return getName();
238    }
239
240    /**
241     * Returns the length of the memory mapped region.
242     *
243     * @return the length of the mapped region
244     */
245    public int getRegionLength() {
246        return regionLength;
247    }
248
249    /**
250     * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
251     * {@code false} otherwise.
252     *
253     * @return whether each write should be force-sync'ed
254     */
255    public boolean isImmediateFlush() {
256        return immediateFlush;
257    }
258
259    /**
260     * Gets this FileManager's content format specified by:
261     * <p>
262     * Key: "fileURI" Value: provided "advertiseURI" param.
263     * </p>
264     *
265     * @return Map of content format keys supporting FileManager
266     */
267    @Override
268    public Map<String, String> getContentFormat() {
269        final Map<String, String> result = new HashMap<>(super.getContentFormat());
270        result.put("fileURI", advertiseURI);
271        return result;
272    }
273
274    @Override
275    protected void flushBuffer(final ByteBuffer buffer) {
276        // do nothing (do not call drain() to avoid spurious remapping)
277    }
278
279    @Override
280    public ByteBuffer getByteBuffer() {
281        return mappedBuffer;
282    }
283
284    @Override
285    public ByteBuffer drain(final ByteBuffer buf) {
286        remap();
287        return mappedBuffer;
288    }
289
290    /**
291     * Factory Data.
292     */
293    private static class FactoryData {
294        private final boolean append;
295        private final boolean immediateFlush;
296        private final int regionLength;
297        private final String advertiseURI;
298        private final Layout<? extends Serializable> layout;
299
300        /**
301         * Constructor.
302         *
303         * @param append Append to existing file or truncate.
304         * @param immediateFlush forces the memory content to be written to the storage device on every event
305         * @param regionLength length of the mapped region
306         * @param advertiseURI the URI to use when advertising the file
307         * @param layout The layout.
308         */
309        public FactoryData(final boolean append, final boolean immediateFlush, final int regionLength,
310                final String advertiseURI, final Layout<? extends Serializable> layout) {
311            this.append = append;
312            this.immediateFlush = immediateFlush;
313            this.regionLength = regionLength;
314            this.advertiseURI = advertiseURI;
315            this.layout = layout;
316        }
317    }
318
319    /**
320     * Factory to create a MemoryMappedFileManager.
321     */
322    private static class MemoryMappedFileManagerFactory
323            implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
324
325        /**
326         * Create a MemoryMappedFileManager.
327         *
328         * @param name The name of the File.
329         * @param data The FactoryData
330         * @return The MemoryMappedFileManager for the File.
331         */
332        @SuppressWarnings("resource")
333        @Override
334        public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
335            final File file = new File(name);
336            if (!data.append) {
337                file.delete();
338            }
339
340            final boolean writeHeader = !data.append || !file.exists();
341            final OutputStream os = NullOutputStream.getInstance();
342            RandomAccessFile raf = null;
343            try {
344                FileUtils.makeParentDirs(file);
345                raf = new RandomAccessFile(name, "rw");
346                final long position = (data.append) ? raf.length() : 0;
347                raf.setLength(position + data.regionLength);
348                return new MemoryMappedFileManager(raf, name, os, data.immediateFlush, position, data.regionLength,
349                        data.advertiseURI, data.layout, writeHeader);
350            } catch (final Exception ex) {
351                LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
352                Closer.closeSilently(raf);
353            }
354            return null;
355        }
356    }
357}