001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache license, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the license for the specific language governing permissions and 015 * limitations under the license. 016 */ 017package org.apache.logging.log4j.core.layout; 018 019import java.io.ByteArrayOutputStream; 020import java.io.IOException; 021import java.io.OutputStream; 022import java.io.PrintWriter; 023import java.io.StringWriter; 024import java.nio.charset.StandardCharsets; 025import java.util.Collections; 026import java.util.Map; 027import java.util.zip.DeflaterOutputStream; 028import java.util.zip.GZIPOutputStream; 029 030import org.apache.logging.log4j.Level; 031import org.apache.logging.log4j.core.Layout; 032import org.apache.logging.log4j.core.LogEvent; 033import org.apache.logging.log4j.core.config.Configuration; 034import org.apache.logging.log4j.core.config.Node; 035import org.apache.logging.log4j.core.config.plugins.Plugin; 036import org.apache.logging.log4j.core.config.plugins.PluginAttribute; 037import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; 038import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; 039import org.apache.logging.log4j.core.config.plugins.PluginElement; 040import org.apache.logging.log4j.core.lookup.StrSubstitutor; 041import org.apache.logging.log4j.core.net.Severity; 042import org.apache.logging.log4j.core.util.JsonUtils; 043import org.apache.logging.log4j.core.util.KeyValuePair; 044import org.apache.logging.log4j.core.util.NetUtils; 045import org.apache.logging.log4j.message.Message; 046import org.apache.logging.log4j.status.StatusLogger; 047import org.apache.logging.log4j.util.StringBuilderFormattable; 048import org.apache.logging.log4j.util.Strings; 049import org.apache.logging.log4j.util.TriConsumer; 050 051/** 052 * Lays out events in the Graylog Extended Log Format (GELF) 1.1. 053 * <p> 054 * This layout compresses JSON to GZIP or ZLIB (the {@code compressionType}) if 055 * log event data is larger than 1024 bytes (the {@code compressionThreshold}). 056 * This layout does not implement chunking. 057 * </p> 058 * 059 * @see <a href="http://docs.graylog.org/en/latest/pages/gelf.html#gelf">GELF specification</a> 060 */ 061@Plugin(name = "GelfLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) 062public final class GelfLayout extends AbstractStringLayout { 063 064 public enum CompressionType { 065 066 GZIP { 067 @Override 068 public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException { 069 return new GZIPOutputStream(os); 070 } 071 }, 072 ZLIB { 073 @Override 074 public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException { 075 return new DeflaterOutputStream(os); 076 } 077 }, 078 OFF { 079 @Override 080 public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException { 081 return null; 082 } 083 }; 084 085 public abstract DeflaterOutputStream createDeflaterOutputStream(OutputStream os) throws IOException; 086 } 087 088 private static final char C = ','; 089 private static final int COMPRESSION_THRESHOLD = 1024; 090 private static final char Q = '\"'; 091 private static final String QC = "\","; 092 private static final String QU = "\"_"; 093 094 private final KeyValuePair[] additionalFields; 095 private final int compressionThreshold; 096 private final CompressionType compressionType; 097 private final String host; 098 private final boolean includeStacktrace; 099 private final boolean includeThreadContext; 100 private final boolean includeNullDelimiter; 101 102 public static class Builder<B extends Builder<B>> extends AbstractStringLayout.Builder<B> 103 implements org.apache.logging.log4j.core.util.Builder<GelfLayout> { 104 105 @PluginBuilderAttribute 106 private String host; 107 108 @PluginElement("AdditionalField") 109 private KeyValuePair[] additionalFields; 110 111 @PluginBuilderAttribute 112 private CompressionType compressionType = CompressionType.GZIP; 113 114 @PluginBuilderAttribute 115 private int compressionThreshold = COMPRESSION_THRESHOLD; 116 117 @PluginBuilderAttribute 118 private boolean includeStacktrace = true; 119 120 @PluginBuilderAttribute 121 private boolean includeThreadContext = true; 122 123 @PluginBuilderAttribute 124 private boolean includeNullDelimiter = false; 125 126 public Builder() { 127 super(); 128 setCharset(StandardCharsets.UTF_8); 129 } 130 131 @Override 132 public GelfLayout build() { 133 return new GelfLayout(getConfiguration(), host, additionalFields, compressionType, compressionThreshold, 134 includeStacktrace, includeThreadContext, includeNullDelimiter); 135 } 136 137 public String getHost() { 138 return host; 139 } 140 141 public CompressionType getCompressionType() { 142 return compressionType; 143 } 144 145 public int getCompressionThreshold() { 146 return compressionThreshold; 147 } 148 149 public boolean isIncludeStacktrace() { 150 return includeStacktrace; 151 } 152 153 public boolean isIncludeThreadContext() { 154 return includeThreadContext; 155 } 156 157 public boolean isIncludeNullDelimiter() { return includeNullDelimiter; } 158 159 public KeyValuePair[] getAdditionalFields() { 160 return additionalFields; 161 } 162 163 /** 164 * The value of the <code>host</code> property (optional, defaults to local host name). 165 * 166 * @return this builder 167 */ 168 public B setHost(final String host) { 169 this.host = host; 170 return asBuilder(); 171 } 172 173 /** 174 * Compression to use (optional, defaults to GZIP). 175 * 176 * @return this builder 177 */ 178 public B setCompressionType(final CompressionType compressionType) { 179 this.compressionType = compressionType; 180 return asBuilder(); 181 } 182 183 /** 184 * Compress if data is larger than this number of bytes (optional, defaults to 1024). 185 * 186 * @return this builder 187 */ 188 public B setCompressionThreshold(final int compressionThreshold) { 189 this.compressionThreshold = compressionThreshold; 190 return asBuilder(); 191 } 192 193 /** 194 * Whether to include full stacktrace of logged Throwables (optional, default to true). 195 * If set to false, only the class name and message of the Throwable will be included. 196 * 197 * @return this builder 198 */ 199 public B setIncludeStacktrace(final boolean includeStacktrace) { 200 this.includeStacktrace = includeStacktrace; 201 return asBuilder(); 202 } 203 204 /** 205 * Whether to include thread context as additional fields (optional, default to true). 206 * 207 * @return this builder 208 */ 209 public B setIncludeThreadContext(final boolean includeThreadContext) { 210 this.includeThreadContext = includeThreadContext; 211 return asBuilder(); 212 } 213 214 /** 215 * Whether to include NULL byte as delimiter after each event (optional, default to false). 216 * Useful for Graylog GELF TCP input. 217 * 218 * @return this builder 219 */ 220 public B setIncludeNullDelimiter(final boolean includeNullDelimiter) { 221 this.includeNullDelimiter = includeNullDelimiter; 222 return asBuilder(); 223 } 224 225 /** 226 * Additional fields to set on each log event. 227 * 228 * @return this builder 229 */ 230 public B setAdditionalFields(final KeyValuePair[] additionalFields) { 231 this.additionalFields = additionalFields; 232 return asBuilder(); 233 } 234 } 235 236 /** 237 * @deprecated Use {@link #newBuilder()} instead 238 */ 239 @Deprecated 240 public GelfLayout(final String host, final KeyValuePair[] additionalFields, final CompressionType compressionType, 241 final int compressionThreshold, final boolean includeStacktrace) { 242 this(null, host, additionalFields, compressionType, compressionThreshold, includeStacktrace, true, false); 243 } 244 245 private GelfLayout(final Configuration config, final String host, final KeyValuePair[] additionalFields, final CompressionType compressionType, 246 final int compressionThreshold, final boolean includeStacktrace, final boolean includeThreadContext, final boolean includeNullDelimiter) { 247 super(config, StandardCharsets.UTF_8, null, null); 248 this.host = host != null ? host : NetUtils.getLocalHostname(); 249 this.additionalFields = additionalFields != null ? additionalFields : new KeyValuePair[0]; 250 if (config == null) { 251 for (final KeyValuePair additionalField : this.additionalFields) { 252 if (valueNeedsLookup(additionalField.getValue())) { 253 throw new IllegalArgumentException("configuration needs to be set when there are additional fields with variables"); 254 } 255 } 256 } 257 this.compressionType = compressionType; 258 this.compressionThreshold = compressionThreshold; 259 this.includeStacktrace = includeStacktrace; 260 this.includeThreadContext = includeThreadContext; 261 this.includeNullDelimiter = includeNullDelimiter; 262 if (includeNullDelimiter && compressionType != CompressionType.OFF) { 263 throw new IllegalArgumentException("null delimiter cannot be used with compression"); 264 } 265 } 266 267 /** 268 * @deprecated Use {@link #newBuilder()} instead 269 */ 270 @Deprecated 271 public static GelfLayout createLayout( 272 //@formatter:off 273 @PluginAttribute("host") final String host, 274 @PluginElement("AdditionalField") final KeyValuePair[] additionalFields, 275 @PluginAttribute(value = "compressionType", 276 defaultString = "GZIP") final CompressionType compressionType, 277 @PluginAttribute(value = "compressionThreshold", 278 defaultInt = COMPRESSION_THRESHOLD) final int compressionThreshold, 279 @PluginAttribute(value = "includeStacktrace", 280 defaultBoolean = true) final boolean includeStacktrace) { 281 // @formatter:on 282 return new GelfLayout(null, host, additionalFields, compressionType, compressionThreshold, includeStacktrace, true, false); 283 } 284 285 @PluginBuilderFactory 286 public static <B extends Builder<B>> B newBuilder() { 287 return new Builder<B>().asBuilder(); 288 } 289 290 @Override 291 public Map<String, String> getContentFormat() { 292 return Collections.emptyMap(); 293 } 294 295 @Override 296 public String getContentType() { 297 return JsonLayout.CONTENT_TYPE + "; charset=" + this.getCharset(); 298 } 299 300 @Override 301 public byte[] toByteArray(final LogEvent event) { 302 final StringBuilder text = toText(event, getStringBuilder(), false); 303 final byte[] bytes = getBytes(text.toString()); 304 return compressionType != CompressionType.OFF && bytes.length > compressionThreshold ? compress(bytes) : bytes; 305 } 306 307 @Override 308 public void encode(final LogEvent event, final ByteBufferDestination destination) { 309 if (compressionType != CompressionType.OFF) { 310 super.encode(event, destination); 311 return; 312 } 313 final StringBuilder text = toText(event, getStringBuilder(), true); 314 final Encoder<StringBuilder> helper = getStringBuilderEncoder(); 315 helper.encode(text, destination); 316 } 317 318 private byte[] compress(final byte[] bytes) { 319 try { 320 final ByteArrayOutputStream baos = new ByteArrayOutputStream(compressionThreshold / 8); 321 try (final DeflaterOutputStream stream = compressionType.createDeflaterOutputStream(baos)) { 322 if (stream == null) { 323 return bytes; 324 } 325 stream.write(bytes); 326 stream.finish(); 327 } 328 return baos.toByteArray(); 329 } catch (final IOException e) { 330 StatusLogger.getLogger().error(e); 331 return bytes; 332 } 333 } 334 335 @Override 336 public String toSerializable(final LogEvent event) { 337 final StringBuilder text = toText(event, getStringBuilder(), false); 338 return text.toString(); 339 } 340 341 private StringBuilder toText(final LogEvent event, final StringBuilder builder, final boolean gcFree) { 342 builder.append('{'); 343 builder.append("\"version\":\"1.1\","); 344 builder.append("\"host\":\""); 345 JsonUtils.quoteAsString(toNullSafeString(host), builder); 346 builder.append(QC); 347 builder.append("\"timestamp\":").append(formatTimestamp(event.getTimeMillis())).append(C); 348 builder.append("\"level\":").append(formatLevel(event.getLevel())).append(C); 349 if (event.getThreadName() != null) { 350 builder.append("\"_thread\":\""); 351 JsonUtils.quoteAsString(event.getThreadName(), builder); 352 builder.append(QC); 353 } 354 if (event.getLoggerName() != null) { 355 builder.append("\"_logger\":\""); 356 JsonUtils.quoteAsString(event.getLoggerName(), builder); 357 builder.append(QC); 358 } 359 if (additionalFields.length > 0) { 360 final StrSubstitutor strSubstitutor = getConfiguration().getStrSubstitutor(); 361 for (final KeyValuePair additionalField : additionalFields) { 362 builder.append(QU); 363 JsonUtils.quoteAsString(additionalField.getKey(), builder); 364 builder.append("\":\""); 365 final String value = valueNeedsLookup(additionalField.getValue()) 366 ? strSubstitutor.replace(event, additionalField.getValue()) 367 : additionalField.getValue(); 368 JsonUtils.quoteAsString(toNullSafeString(value), builder); 369 builder.append(QC); 370 } 371 } 372 if (includeThreadContext) { 373 event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder); 374 } 375 if (event.getThrown() != null) { 376 builder.append("\"full_message\":\""); 377 if (includeStacktrace) { 378 JsonUtils.quoteAsString(formatThrowable(event.getThrown()), builder); 379 } else { 380 JsonUtils.quoteAsString(event.getThrown().toString(), builder); 381 } 382 builder.append(QC); 383 } 384 385 builder.append("\"short_message\":\""); 386 final Message message = event.getMessage(); 387 if (message instanceof CharSequence) { 388 JsonUtils.quoteAsString(((CharSequence)message), builder); 389 } else if (gcFree && message instanceof StringBuilderFormattable) { 390 final StringBuilder messageBuffer = getMessageStringBuilder(); 391 try { 392 ((StringBuilderFormattable) message).formatTo(messageBuffer); 393 JsonUtils.quoteAsString(messageBuffer, builder); 394 } finally { 395 trimToMaxSize(messageBuffer); 396 } 397 } else { 398 JsonUtils.quoteAsString(toNullSafeString(message.getFormattedMessage()), builder); 399 } 400 builder.append(Q); 401 builder.append('}'); 402 if (includeNullDelimiter) { 403 builder.append('\0'); 404 } 405 return builder; 406 } 407 408 private static boolean valueNeedsLookup(final String value) { 409 return value != null && value.contains("${"); 410 } 411 412 private static final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() { 413 @Override 414 public void accept(final String key, final Object value, final StringBuilder stringBuilder) { 415 stringBuilder.append(QU); 416 JsonUtils.quoteAsString(key, stringBuilder); 417 stringBuilder.append("\":\""); 418 JsonUtils.quoteAsString(toNullSafeString(String.valueOf(value)), stringBuilder); 419 stringBuilder.append(QC); 420 } 421 }; 422 423 private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<>(); 424 425 private static StringBuilder getMessageStringBuilder() { 426 StringBuilder result = messageStringBuilder.get(); 427 if (result == null) { 428 result = new StringBuilder(DEFAULT_STRING_BUILDER_SIZE); 429 messageStringBuilder.set(result); 430 } 431 result.setLength(0); 432 return result; 433 } 434 435 private static CharSequence toNullSafeString(final CharSequence s) { 436 return s == null ? Strings.EMPTY : s; 437 } 438 439 /** 440 * Non-private to make it accessible from unit test. 441 */ 442 static CharSequence formatTimestamp(final long timeMillis) { 443 if (timeMillis < 1000) { 444 return "0"; 445 } 446 final StringBuilder builder = getTimestampStringBuilder(); 447 builder.append(timeMillis); 448 builder.insert(builder.length() - 3, '.'); 449 return builder; 450 } 451 452 private static final ThreadLocal<StringBuilder> timestampStringBuilder = new ThreadLocal<>(); 453 454 private static StringBuilder getTimestampStringBuilder() { 455 StringBuilder result = timestampStringBuilder.get(); 456 if (result == null) { 457 result = new StringBuilder(20); 458 timestampStringBuilder.set(result); 459 } 460 result.setLength(0); 461 return result; 462 } 463 464 /** 465 * http://en.wikipedia.org/wiki/Syslog#Severity_levels 466 */ 467 private int formatLevel(final Level level) { 468 return Severity.getSeverity(level).getCode(); 469 } 470 471 /** 472 * Non-private to make it accessible from unit test. 473 */ 474 static CharSequence formatThrowable(final Throwable throwable) { 475 // stack traces are big enough to provide a reasonably large initial capacity here 476 final StringWriter sw = new StringWriter(2048); 477 final PrintWriter pw = new PrintWriter(sw); 478 throwable.printStackTrace(pw); 479 pw.flush(); 480 return sw.getBuffer(); 481 } 482}