Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.appender;
018
019import java.io.File;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.RandomAccessFile;
023import java.io.Serializable;
024import java.lang.reflect.Method;
025import java.nio.ByteOrder;
026import java.nio.MappedByteBuffer;
027import java.nio.channels.FileChannel;
028import java.security.AccessController;
029import java.security.PrivilegedActionException;
030import java.security.PrivilegedExceptionAction;
031import java.util.HashMap;
032import java.util.Map;
033
034import org.apache.logging.log4j.core.Layout;
035import org.apache.logging.log4j.core.util.Assert;
036import org.apache.logging.log4j.core.util.Closer;
037import org.apache.logging.log4j.core.util.NullOutputStream;
038
039/**
040 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
041 * memory and writes to this memory region.
042 * <p>
043 * 
044 * @see <a
045 *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
046 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
047 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
048 * @see <a
049 *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
050 * 
051 * @since 2.1
052 */
053public class MemoryMappedFileManager extends OutputStreamManager {
054    static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
055    private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
056
057    private final boolean isForce;
058    private final int regionLength;
059    private final String advertiseURI;
060    private final RandomAccessFile randomAccessFile;
061    private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
062    private MappedByteBuffer mappedBuffer;
063    private long mappingOffset;
064
065    protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
066            final boolean force, final long position, final int regionLength, final String advertiseURI,
067            final Layout<? extends Serializable> layout) throws IOException {
068        super(os, fileName, layout);
069        this.isForce = force;
070        this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
071        this.regionLength = regionLength;
072        this.advertiseURI = advertiseURI;
073        this.isEndOfBatch.set(Boolean.FALSE);
074        this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
075        this.mappingOffset = position;
076    }
077
078    /**
079     * Returns the MemoryMappedFileManager.
080     *
081     * @param fileName The name of the file to manage.
082     * @param append true if the file should be appended to, false if it should be overwritten.
083     * @param isForce true if the contents should be flushed to disk on every write
084     * @param regionLength The mapped region length.
085     * @param advertiseURI the URI to use when advertising the file
086     * @param layout The layout.
087     * @return A MemoryMappedFileManager for the File.
088     */
089    public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
090            final boolean isForce, final int regionLength, final String advertiseURI,
091            final Layout<? extends Serializable> layout) {
092        return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
093                advertiseURI, layout), FACTORY);
094    }
095
096    public Boolean isEndOfBatch() {
097        return isEndOfBatch.get();
098    }
099
100    public void setEndOfBatch(final boolean isEndOfBatch) {
101        this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
102    }
103
104    @Override
105    protected synchronized void write(final byte[] bytes, int offset, int length) {
106        super.write(bytes, offset, length); // writes to dummy output stream
107
108        while (length > mappedBuffer.remaining()) {
109            final int chunk = mappedBuffer.remaining();
110            mappedBuffer.put(bytes, offset, chunk);
111            offset += chunk;
112            length -= chunk;
113            remap();
114        }
115        mappedBuffer.put(bytes, offset, length);
116
117        // no need to call flush() if force is true,
118        // already done in AbstractOutputStreamAppender.append
119    }
120
121    private synchronized void remap() {
122        final long offset = this.mappingOffset + mappedBuffer.position();
123        final int length = mappedBuffer.remaining() + regionLength;
124        try {
125            unsafeUnmap(mappedBuffer);
126            final long fileLength = randomAccessFile.length() + regionLength;
127            LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
128
129            long startNanos = System.nanoTime();
130            randomAccessFile.setLength(fileLength);
131            final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
132            LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
133
134            mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
135            mappingOffset = offset;
136        } catch (final Exception ex) {
137            LOGGER.error("Unable to remap " + getName() + ". " + ex);
138        }
139    }
140
141    @Override
142    public synchronized void flush() {
143        mappedBuffer.force();
144    }
145
146    @Override
147    public synchronized void close() {
148        final long position = mappedBuffer.position();
149        final long length = mappingOffset + position;
150        try {
151            unsafeUnmap(mappedBuffer);
152        } catch (final Exception ex) {
153            LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
154        }
155        try {
156            LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
157                    length, mappingOffset, position);
158            randomAccessFile.setLength(length);
159            randomAccessFile.close();
160        } catch (final IOException ex) {
161            LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
162        }
163    }
164
165    public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
166            final int size) throws IOException {
167        for (int i = 1;; i++) {
168            try {
169                LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
170
171                final long startNanos = System.nanoTime();
172                final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
173                map.order(ByteOrder.nativeOrder());
174
175                final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
176                LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
177
178                return map;
179            } catch (final IOException e) {
180                if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
181                    throw e;
182                }
183                LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
184                if (i < 10) {
185                    Thread.yield();
186                } else {
187                    try {
188                        Thread.sleep(1);
189                    } catch (final InterruptedException ignored) {
190                        Thread.currentThread().interrupt();
191                        throw e;
192                    }
193                }
194            }
195        }
196    }
197
198    private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
199        LOGGER.debug("MMapAppender unmapping old buffer...");
200        final long startNanos = System.nanoTime();
201        AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
202            @Override
203            public Object run() throws Exception {
204                final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
205                getCleanerMethod.setAccessible(true);
206                final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
207                final Method cleanMethod = cleaner.getClass().getMethod("clean");
208                cleanMethod.invoke(cleaner);
209                return null;
210            }
211        });
212        final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
213        LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
214    }
215
216    /**
217     * Returns the name of the File being managed.
218     *
219     * @return The name of the File being managed.
220     */
221    public String getFileName() {
222        return getName();
223    }
224
225    /**
226     * Returns the length of the memory mapped region.
227     * 
228     * @return the length of the mapped region
229     */
230    public int getRegionLength() {
231        return regionLength;
232    }
233
234    /**
235     * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
236     * {@code false} otherwise.
237     * 
238     * @return whether each write should be force-sync'ed
239     */
240    public boolean isImmediateFlush() {
241        return isForce;
242    }
243
244    /**
245     * Gets this FileManager's content format specified by:
246     * <p>
247     * Key: "fileURI" Value: provided "advertiseURI" param.
248     * </p>
249     * 
250     * @return Map of content format keys supporting FileManager
251     */
252    @Override
253    public Map<String, String> getContentFormat() {
254        final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
255        result.put("fileURI", advertiseURI);
256        return result;
257    }
258
259    /**
260     * Factory Data.
261     */
262    private static class FactoryData {
263        private final boolean append;
264        private final boolean force;
265        private final int regionLength;
266        private final String advertiseURI;
267        private final Layout<? extends Serializable> layout;
268
269        /**
270         * Constructor.
271         *
272         * @param append Append to existing file or truncate.
273         * @param force forces the memory content to be written to the storage device on every event
274         * @param regionLength length of the mapped region
275         */
276        public FactoryData(final boolean append, final boolean force, final int regionLength,
277                final String advertiseURI, final Layout<? extends Serializable> layout) {
278            this.append = append;
279            this.force = force;
280            this.regionLength = regionLength;
281            this.advertiseURI = advertiseURI;
282            this.layout = layout;
283        }
284    }
285
286    /**
287     * Factory to create a MemoryMappedFileManager.
288     */
289    private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
290
291        /**
292         * Create a MemoryMappedFileManager.
293         *
294         * @param name The name of the File.
295         * @param data The FactoryData
296         * @return The MemoryMappedFileManager for the File.
297         */
298        @SuppressWarnings("resource")
299        @Override
300        public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
301            final File file = new File(name);
302            final File parent = file.getParentFile();
303            if (null != parent && !parent.exists()) {
304                parent.mkdirs();
305            }
306            if (!data.append) {
307                file.delete();
308            }
309
310            final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
311            RandomAccessFile raf = null;
312            try {
313                raf = new RandomAccessFile(name, "rw");
314                final long position = (data.append) ? raf.length() : 0;
315                raf.setLength(position + data.regionLength);
316                return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
317                        data.advertiseURI, data.layout);
318            } catch (final Exception ex) {
319                LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
320                Closer.closeSilently(raf);
321            }
322            return null;
323        }
324    }
325}