View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache license, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License. You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the license for the specific language governing permissions and
15   * limitations under the license.
16   */
17  package org.apache.logging.log4j.core.appender;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.io.RandomAccessFile;
23  import java.io.Serializable;
24  import java.lang.reflect.Method;
25  import java.nio.ByteOrder;
26  import java.nio.MappedByteBuffer;
27  import java.nio.channels.FileChannel;
28  import java.security.AccessController;
29  import java.security.PrivilegedActionException;
30  import java.security.PrivilegedExceptionAction;
31  import java.util.HashMap;
32  import java.util.Map;
33  
34  import org.apache.logging.log4j.core.Layout;
35  import org.apache.logging.log4j.core.util.Assert;
36  import org.apache.logging.log4j.core.util.Closer;
37  import org.apache.logging.log4j.core.util.NullOutputStream;
38  
39  /**
40   * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
41   * memory and writes to this memory region.
42   * <p>
43   * 
44   * @see <a
45   *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
46   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
47   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
48   * @see <a
49   *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
50   * 
51   * @since 2.1
52   */
53  public class MemoryMappedFileManager extends OutputStreamManager {
54      static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
55      private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
56  
57      private final boolean isForce;
58      private final int regionLength;
59      private final String advertiseURI;
60      private final RandomAccessFile randomAccessFile;
61      private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
62      private MappedByteBuffer mappedBuffer;
63      private long mappingOffset;
64  
65      protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
66              final boolean force, final long position, final int regionLength, final String advertiseURI,
67              final Layout<? extends Serializable> layout) throws IOException {
68          super(os, fileName, layout);
69          this.isForce = force;
70          this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
71          this.regionLength = regionLength;
72          this.advertiseURI = advertiseURI;
73          this.isEndOfBatch.set(Boolean.FALSE);
74          this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
75          this.mappingOffset = position;
76      }
77  
78      /**
79       * Returns the MemoryMappedFileManager.
80       *
81       * @param fileName The name of the file to manage.
82       * @param append true if the file should be appended to, false if it should be overwritten.
83       * @param isForce true if the contents should be flushed to disk on every write
84       * @param regionLength The mapped region length.
85       * @param advertiseURI the URI to use when advertising the file
86       * @param layout The layout.
87       * @return A MemoryMappedFileManager for the File.
88       */
89      public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
90              final boolean isForce, final int regionLength, final String advertiseURI,
91              final Layout<? extends Serializable> layout) {
92          return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
93                  advertiseURI, layout), FACTORY);
94      }
95  
96      public Boolean isEndOfBatch() {
97          return isEndOfBatch.get();
98      }
99  
100     public void setEndOfBatch(final boolean isEndOfBatch) {
101         this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
102     }
103 
104     @Override
105     protected synchronized void write(final byte[] bytes, int offset, int length) {
106         super.write(bytes, offset, length); // writes to dummy output stream
107 
108         while (length > mappedBuffer.remaining()) {
109             final int chunk = mappedBuffer.remaining();
110             mappedBuffer.put(bytes, offset, chunk);
111             offset += chunk;
112             length -= chunk;
113             remap();
114         }
115         mappedBuffer.put(bytes, offset, length);
116 
117         // no need to call flush() if force is true,
118         // already done in AbstractOutputStreamAppender.append
119     }
120 
121     private synchronized void remap() {
122         final long offset = this.mappingOffset + mappedBuffer.position();
123         final int length = mappedBuffer.remaining() + regionLength;
124         try {
125             unsafeUnmap(mappedBuffer);
126             final long fileLength = randomAccessFile.length() + regionLength;
127             LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
128 
129             long startNanos = System.nanoTime();
130             randomAccessFile.setLength(fileLength);
131             final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
132             LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
133 
134             mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
135             mappingOffset = offset;
136         } catch (final Exception ex) {
137             LOGGER.error("Unable to remap " + getName() + ". " + ex);
138         }
139     }
140 
141     @Override
142     public synchronized void flush() {
143         mappedBuffer.force();
144     }
145 
146     @Override
147     public synchronized void close() {
148         final long position = mappedBuffer.position();
149         final long length = mappingOffset + position;
150         try {
151             unsafeUnmap(mappedBuffer);
152         } catch (final Exception ex) {
153             LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
154         }
155         try {
156             LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
157                     length, mappingOffset, position);
158             randomAccessFile.setLength(length);
159             randomAccessFile.close();
160         } catch (final IOException ex) {
161             LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
162         }
163     }
164 
165     public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
166             final int size) throws IOException {
167         for (int i = 1;; i++) {
168             try {
169                 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
170 
171                 final long startNanos = System.nanoTime();
172                 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
173                 map.order(ByteOrder.nativeOrder());
174 
175                 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
176                 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
177 
178                 return map;
179             } catch (final IOException e) {
180                 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
181                     throw e;
182                 }
183                 LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
184                 if (i < 10) {
185                     Thread.yield();
186                 } else {
187                     try {
188                         Thread.sleep(1);
189                     } catch (final InterruptedException ignored) {
190                         Thread.currentThread().interrupt();
191                         throw e;
192                     }
193                 }
194             }
195         }
196     }
197 
198     private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
199         LOGGER.debug("MMapAppender unmapping old buffer...");
200         final long startNanos = System.nanoTime();
201         AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
202             @Override
203             public Object run() throws Exception {
204                 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
205                 getCleanerMethod.setAccessible(true);
206                 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
207                 final Method cleanMethod = cleaner.getClass().getMethod("clean");
208                 cleanMethod.invoke(cleaner);
209                 return null;
210             }
211         });
212         final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
213         LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
214     }
215 
216     /**
217      * Returns the name of the File being managed.
218      *
219      * @return The name of the File being managed.
220      */
221     public String getFileName() {
222         return getName();
223     }
224 
225     /**
226      * Returns the length of the memory mapped region.
227      * 
228      * @return the length of the mapped region
229      */
230     public int getRegionLength() {
231         return regionLength;
232     }
233 
234     /**
235      * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
236      * {@code false} otherwise.
237      * 
238      * @return whether each write should be force-sync'ed
239      */
240     public boolean isImmediateFlush() {
241         return isForce;
242     }
243 
244     /**
245      * Gets this FileManager's content format specified by:
246      * <p>
247      * Key: "fileURI" Value: provided "advertiseURI" param.
248      * </p>
249      * 
250      * @return Map of content format keys supporting FileManager
251      */
252     @Override
253     public Map<String, String> getContentFormat() {
254         final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
255         result.put("fileURI", advertiseURI);
256         return result;
257     }
258 
259     /**
260      * Factory Data.
261      */
262     private static class FactoryData {
263         private final boolean append;
264         private final boolean force;
265         private final int regionLength;
266         private final String advertiseURI;
267         private final Layout<? extends Serializable> layout;
268 
269         /**
270          * Constructor.
271          *
272          * @param append Append to existing file or truncate.
273          * @param force forces the memory content to be written to the storage device on every event
274          * @param regionLength length of the mapped region
275          */
276         public FactoryData(final boolean append, final boolean force, final int regionLength,
277                 final String advertiseURI, final Layout<? extends Serializable> layout) {
278             this.append = append;
279             this.force = force;
280             this.regionLength = regionLength;
281             this.advertiseURI = advertiseURI;
282             this.layout = layout;
283         }
284     }
285 
286     /**
287      * Factory to create a MemoryMappedFileManager.
288      */
289     private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
290 
291         /**
292          * Create a MemoryMappedFileManager.
293          *
294          * @param name The name of the File.
295          * @param data The FactoryData
296          * @return The MemoryMappedFileManager for the File.
297          */
298         @SuppressWarnings("resource")
299         @Override
300         public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
301             final File file = new File(name);
302             final File parent = file.getParentFile();
303             if (null != parent && !parent.exists()) {
304                 parent.mkdirs();
305             }
306             if (!data.append) {
307                 file.delete();
308             }
309 
310             final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
311             RandomAccessFile raf = null;
312             try {
313                 raf = new RandomAccessFile(name, "rw");
314                 final long position = (data.append) ? raf.length() : 0;
315                 raf.setLength(position + data.regionLength);
316                 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
317                         data.advertiseURI, data.layout);
318             } catch (final Exception ex) {
319                 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
320                 Closer.closeSilently(raf);
321             }
322             return null;
323         }
324     }
325 }