001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.layout;
018
019import java.io.ByteArrayOutputStream;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.PrintWriter;
023import java.io.StringWriter;
024import java.nio.charset.StandardCharsets;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.List;
028import java.util.Map;
029import java.util.Objects;
030import java.util.zip.DeflaterOutputStream;
031import java.util.zip.GZIPOutputStream;
032
033import org.apache.logging.log4j.Level;
034import org.apache.logging.log4j.core.Layout;
035import org.apache.logging.log4j.core.LogEvent;
036import org.apache.logging.log4j.core.config.Configuration;
037import org.apache.logging.log4j.core.layout.internal.ExcludeChecker;
038import org.apache.logging.log4j.core.layout.internal.IncludeChecker;
039import org.apache.logging.log4j.core.layout.internal.ListChecker;
040import org.apache.logging.log4j.core.config.Node;
041import org.apache.logging.log4j.core.config.plugins.Plugin;
042import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
043import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute;
044import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
045import org.apache.logging.log4j.core.config.plugins.PluginElement;
046import org.apache.logging.log4j.core.lookup.StrSubstitutor;
047import org.apache.logging.log4j.core.net.Severity;
048import org.apache.logging.log4j.core.util.JsonUtils;
049import org.apache.logging.log4j.core.util.KeyValuePair;
050import org.apache.logging.log4j.core.util.NetUtils;
051import org.apache.logging.log4j.core.util.Patterns;
052import org.apache.logging.log4j.message.Message;
053import org.apache.logging.log4j.status.StatusLogger;
054import org.apache.logging.log4j.util.StringBuilderFormattable;
055import org.apache.logging.log4j.util.Strings;
056import org.apache.logging.log4j.util.TriConsumer;
057
058/**
059 * Lays out events in the Graylog Extended Log Format (GELF) 1.1.
060 * <p>
061 * This layout compresses JSON to GZIP or ZLIB (the {@code compressionType}) if
062 * log event data is larger than 1024 bytes (the {@code compressionThreshold}).
063 * This layout does not implement chunking.
064 * </p>
065 *
066 * @see <a href="http://docs.graylog.org/en/latest/pages/gelf.html#gelf">GELF specification</a>
067 */
068@Plugin(name = "GelfLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
069public final class GelfLayout extends AbstractStringLayout {
070
071    public enum CompressionType {
072
073        GZIP {
074            @Override
075            public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException {
076                return new GZIPOutputStream(os);
077            }
078        },
079        ZLIB {
080            @Override
081            public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException {
082                return new DeflaterOutputStream(os);
083            }
084        },
085        OFF {
086            @Override
087            public DeflaterOutputStream createDeflaterOutputStream(final OutputStream os) throws IOException {
088                return null;
089            }
090        };
091
092        public abstract DeflaterOutputStream createDeflaterOutputStream(OutputStream os) throws IOException;
093    }
094
095    private static final char C = ',';
096    private static final int COMPRESSION_THRESHOLD = 1024;
097    private static final char Q = '\"';
098    private static final String QC = "\",";
099    private static final String QU = "\"_";
100
101    private final KeyValuePair[] additionalFields;
102    private final int compressionThreshold;
103    private final CompressionType compressionType;
104    private final String host;
105    private final boolean includeStacktrace;
106    private final boolean includeThreadContext;
107    private final boolean includeNullDelimiter;
108    private final boolean includeNewLineDelimiter;
109    private final PatternLayout layout;
110    private final FieldWriter fieldWriter;
111
112    public static class Builder<B extends Builder<B>> extends AbstractStringLayout.Builder<B>
113        implements org.apache.logging.log4j.core.util.Builder<GelfLayout> {
114
115        @PluginBuilderAttribute
116        private String host;
117
118        @PluginElement("AdditionalField")
119        private KeyValuePair[] additionalFields;
120
121        @PluginBuilderAttribute
122        private CompressionType compressionType = CompressionType.GZIP;
123
124        @PluginBuilderAttribute
125        private int compressionThreshold = COMPRESSION_THRESHOLD;
126
127        @PluginBuilderAttribute
128        private boolean includeStacktrace = true;
129
130        @PluginBuilderAttribute
131        private boolean includeThreadContext = true;
132
133        @PluginBuilderAttribute
134        private boolean includeNullDelimiter = false;
135
136        @PluginBuilderAttribute
137        private boolean includeNewLineDelimiter = false;
138
139        @PluginBuilderAttribute
140        private String threadContextIncludes = null;
141
142        @PluginBuilderAttribute
143        private String threadContextExcludes = null;
144
145        @PluginBuilderAttribute
146        private String messagePattern = null;
147
148
149        public Builder() {
150            setCharset(StandardCharsets.UTF_8);
151        }
152
153        @Override
154        public GelfLayout build() {
155            ListChecker checker = null;
156            if (threadContextExcludes != null) {
157                final String[] array = threadContextExcludes.split(Patterns.COMMA_SEPARATOR);
158                if (array.length > 0) {
159                    List<String> excludes = new ArrayList<>(array.length);
160                    for (final String str : array) {
161                        excludes.add(str.trim());
162                    }
163                    checker = new ExcludeChecker(excludes);
164                }
165            }
166            if (threadContextIncludes != null) {
167                final String[] array = threadContextIncludes.split(Patterns.COMMA_SEPARATOR);
168                if (array.length > 0) {
169                    List<String> includes = new ArrayList<>(array.length);
170                    for (final String str : array) {
171                        includes.add(str.trim());
172                    }
173                    checker = new IncludeChecker(includes);
174                }
175            }
176            if (checker == null) {
177                checker = ListChecker.NOOP_CHECKER;
178            }
179            PatternLayout patternLayout = null;
180            if (messagePattern != null) {
181                patternLayout = PatternLayout.newBuilder().withPattern(messagePattern)
182                        .withAlwaysWriteExceptions(includeStacktrace)
183                        .withConfiguration(getConfiguration())
184                        .build();
185            }
186            return new GelfLayout(getConfiguration(), host, additionalFields, compressionType, compressionThreshold,
187                    includeStacktrace, includeThreadContext, includeNullDelimiter, includeNewLineDelimiter, checker,
188                    patternLayout);
189        }
190
191        public String getHost() {
192            return host;
193        }
194
195        public CompressionType getCompressionType() {
196            return compressionType;
197        }
198
199        public int getCompressionThreshold() {
200            return compressionThreshold;
201        }
202
203        public boolean isIncludeStacktrace() {
204            return includeStacktrace;
205        }
206
207        public boolean isIncludeThreadContext() {
208            return includeThreadContext;
209        }
210
211        public boolean isIncludeNullDelimiter() { return includeNullDelimiter; }
212
213        public boolean isIncludeNewLineDelimiter() {
214            return includeNewLineDelimiter;
215        }
216
217        public KeyValuePair[] getAdditionalFields() {
218            return additionalFields;
219        }
220
221        /**
222         * The value of the <code>host</code> property (optional, defaults to local host name).
223         *
224         * @return this builder
225         */
226        public B setHost(final String host) {
227            this.host = host;
228            return asBuilder();
229        }
230
231        /**
232         * Compression to use (optional, defaults to GZIP).
233         *
234         * @return this builder
235         */
236        public B setCompressionType(final CompressionType compressionType) {
237            this.compressionType = compressionType;
238            return asBuilder();
239        }
240
241        /**
242         * Compress if data is larger than this number of bytes (optional, defaults to 1024).
243         *
244         * @return this builder
245         */
246        public B setCompressionThreshold(final int compressionThreshold) {
247            this.compressionThreshold = compressionThreshold;
248            return asBuilder();
249        }
250
251        /**
252         * Whether to include full stacktrace of logged Throwables (optional, default to true).
253         * If set to false, only the class name and message of the Throwable will be included.
254         *
255         * @return this builder
256         */
257        public B setIncludeStacktrace(final boolean includeStacktrace) {
258            this.includeStacktrace = includeStacktrace;
259            return asBuilder();
260        }
261
262        /**
263         * Whether to include thread context as additional fields (optional, default to true).
264         *
265         * @return this builder
266         */
267        public B setIncludeThreadContext(final boolean includeThreadContext) {
268            this.includeThreadContext = includeThreadContext;
269            return asBuilder();
270        }
271
272        /**
273         * Whether to include NULL byte as delimiter after each event (optional, default to false).
274         * Useful for Graylog GELF TCP input.
275         *
276         * @return this builder
277         */
278        public B setIncludeNullDelimiter(final boolean includeNullDelimiter) {
279            this.includeNullDelimiter = includeNullDelimiter;
280            return asBuilder();
281        }
282
283        /**
284         * Whether to include newline (LF) as delimiter after each event (optional, default to false).
285         *
286         * @return this builder
287         */
288        public B setIncludeNewLineDelimiter(final boolean includeNewLineDelimiter) {
289            this.includeNewLineDelimiter = includeNewLineDelimiter;
290            return asBuilder();
291        }
292
293        /**
294         * Additional fields to set on each log event.
295         *
296         * @return this builder
297         */
298        public B setAdditionalFields(final KeyValuePair[] additionalFields) {
299            this.additionalFields = additionalFields;
300            return asBuilder();
301        }
302
303        /**
304         * The pattern to use to format the message.
305         * @param pattern the pattern string.
306         * @return this builder
307         */
308        public B setMessagePattern(final String pattern) {
309            this.messagePattern = pattern;
310            return asBuilder();
311        }
312
313        /**
314         * A comma separated list of thread context keys to include;
315         * @param mdcIncludes the list of keys.
316         * @return this builder
317         */
318        public B setMdcIncludes(final String mdcIncludes) {
319            this.threadContextIncludes = mdcIncludes;
320            return asBuilder();
321        }
322
323        /**
324         * A comma separated list of thread context keys to include;
325         * @param mdcExcludes the list of keys.
326         * @return this builder
327         */
328        public B setMdcExcludes(final String mdcExcludes) {
329            this.threadContextExcludes = mdcExcludes;
330            return asBuilder();
331        }
332    }
333
334    /**
335     * @deprecated Use {@link #newBuilder()} instead
336     */
337    @Deprecated
338    public GelfLayout(final String host, final KeyValuePair[] additionalFields, final CompressionType compressionType,
339                      final int compressionThreshold, final boolean includeStacktrace) {
340        this(null, host, additionalFields, compressionType, compressionThreshold, includeStacktrace, true, false, false,
341                null, null);
342    }
343
344    private GelfLayout(final Configuration config, final String host, final KeyValuePair[] additionalFields,
345            final CompressionType compressionType, final int compressionThreshold, final boolean includeStacktrace,
346            final boolean includeThreadContext, final boolean includeNullDelimiter, final boolean includeNewLineDelimiter,
347            final ListChecker listChecker, final PatternLayout patternLayout) {
348        super(config, StandardCharsets.UTF_8, null, null);
349        this.host = host != null ? host : NetUtils.getLocalHostname();
350        this.additionalFields = additionalFields != null ? additionalFields : new KeyValuePair[0];
351        if (config == null) {
352            for (final KeyValuePair additionalField : this.additionalFields) {
353                if (valueNeedsLookup(additionalField.getValue())) {
354                    throw new IllegalArgumentException("configuration needs to be set when there are additional fields with variables");
355                }
356            }
357        }
358        this.compressionType = compressionType;
359        this.compressionThreshold = compressionThreshold;
360        this.includeStacktrace = includeStacktrace;
361        this.includeThreadContext = includeThreadContext;
362        this.includeNullDelimiter = includeNullDelimiter;
363        this.includeNewLineDelimiter = includeNewLineDelimiter;
364        if (includeNullDelimiter && compressionType != CompressionType.OFF) {
365            throw new IllegalArgumentException("null delimiter cannot be used with compression");
366        }
367        this.fieldWriter = new FieldWriter(listChecker);
368        this.layout = patternLayout;
369    }
370
371    @Override
372    public String toString() {
373        StringBuilder sb = new StringBuilder();
374        sb.append("host=").append(host);
375        sb.append(", compressionType=").append(compressionType.toString());
376        sb.append(", compressionThreshold=").append(compressionThreshold);
377        sb.append(", includeStackTrace=").append(includeStacktrace);
378        sb.append(", includeThreadContext=").append(includeThreadContext);
379        sb.append(", includeNullDelimiter=").append(includeNullDelimiter);
380        sb.append(", includeNewLineDelimiter=").append(includeNewLineDelimiter);
381        String threadVars = fieldWriter.getChecker().toString();
382        if (threadVars.length() > 0) {
383            sb.append(", ").append(threadVars);
384        }
385        if (layout != null) {
386            sb.append(", PatternLayout{").append(layout.toString()).append("}");
387        }
388        return sb.toString();
389    }
390
391    /**
392     * @deprecated Use {@link #newBuilder()} instead
393     */
394    @Deprecated
395    public static GelfLayout createLayout(
396            //@formatter:off
397            @PluginAttribute("host") final String host,
398            @PluginElement("AdditionalField") final KeyValuePair[] additionalFields,
399            @PluginAttribute(value = "compressionType",
400                defaultString = "GZIP") final CompressionType compressionType,
401            @PluginAttribute(value = "compressionThreshold",
402                defaultInt = COMPRESSION_THRESHOLD) final int compressionThreshold,
403            @PluginAttribute(value = "includeStacktrace",
404                defaultBoolean = true) final boolean includeStacktrace) {
405            // @formatter:on
406        return new GelfLayout(null, host, additionalFields, compressionType, compressionThreshold, includeStacktrace,
407                true, false, false, null, null);
408    }
409
410    @PluginBuilderFactory
411    public static <B extends Builder<B>> B newBuilder() {
412        return new Builder<B>().asBuilder();
413    }
414
415    @Override
416    public Map<String, String> getContentFormat() {
417        return Collections.emptyMap();
418    }
419
420    @Override
421    public String getContentType() {
422        return JsonLayout.CONTENT_TYPE + "; charset=" + this.getCharset();
423    }
424
425    @Override
426    public byte[] toByteArray(final LogEvent event) {
427        final StringBuilder text = toText(event, getStringBuilder(), false);
428        final byte[] bytes = getBytes(text.toString());
429        return compressionType != CompressionType.OFF && bytes.length > compressionThreshold ? compress(bytes) : bytes;
430    }
431
432    @Override
433    public void encode(final LogEvent event, final ByteBufferDestination destination) {
434        if (compressionType != CompressionType.OFF) {
435            super.encode(event, destination);
436            return;
437        }
438        final StringBuilder text = toText(event, getStringBuilder(), true);
439        final Encoder<StringBuilder> helper = getStringBuilderEncoder();
440        helper.encode(text, destination);
441    }
442
443    @Override
444    public boolean requiresLocation() {
445        return Objects.nonNull(layout) && layout.requiresLocation();
446    }
447
448    private byte[] compress(final byte[] bytes) {
449        try {
450            final ByteArrayOutputStream baos = new ByteArrayOutputStream(compressionThreshold / 8);
451            try (final DeflaterOutputStream stream = compressionType.createDeflaterOutputStream(baos)) {
452                if (stream == null) {
453                    return bytes;
454                }
455                stream.write(bytes);
456                stream.finish();
457            }
458            return baos.toByteArray();
459        } catch (final IOException e) {
460            StatusLogger.getLogger().error(e);
461            return bytes;
462        }
463    }
464
465    @Override
466    public String toSerializable(final LogEvent event) {
467        final StringBuilder text = toText(event, getStringBuilder(), false);
468        return text.toString();
469    }
470
471    private StringBuilder toText(final LogEvent event, final StringBuilder builder, final boolean gcFree) {
472        builder.append('{');
473        builder.append("\"version\":\"1.1\",");
474        builder.append("\"host\":\"");
475        JsonUtils.quoteAsString(toNullSafeString(host), builder);
476        builder.append(QC);
477        builder.append("\"timestamp\":").append(formatTimestamp(event.getTimeMillis())).append(C);
478        builder.append("\"level\":").append(formatLevel(event.getLevel())).append(C);
479        if (event.getThreadName() != null) {
480            builder.append("\"_thread\":\"");
481            JsonUtils.quoteAsString(event.getThreadName(), builder);
482            builder.append(QC);
483        }
484        if (event.getLoggerName() != null) {
485            builder.append("\"_logger\":\"");
486            JsonUtils.quoteAsString(event.getLoggerName(), builder);
487            builder.append(QC);
488        }
489        if (additionalFields.length > 0) {
490            final StrSubstitutor strSubstitutor = getConfiguration().getStrSubstitutor();
491            for (final KeyValuePair additionalField : additionalFields) {
492                builder.append(QU);
493                JsonUtils.quoteAsString(additionalField.getKey(), builder);
494                builder.append("\":\"");
495                final String value = valueNeedsLookup(additionalField.getValue())
496                    ? strSubstitutor.replace(event, additionalField.getValue())
497                    : additionalField.getValue();
498                JsonUtils.quoteAsString(toNullSafeString(value), builder);
499                builder.append(QC);
500            }
501        }
502        if (includeThreadContext) {
503            event.getContextData().forEach(fieldWriter, builder);
504        }
505
506        if (event.getThrown() != null || layout != null) {
507            builder.append("\"full_message\":\"");
508            if (layout != null) {
509                final StringBuilder messageBuffer = getMessageStringBuilder();
510                layout.serialize(event, messageBuffer);
511                JsonUtils.quoteAsString(messageBuffer, builder);
512            } else if (includeStacktrace) {
513                JsonUtils.quoteAsString(formatThrowable(event.getThrown()), builder);
514            } else {
515                JsonUtils.quoteAsString(event.getThrown().toString(), builder);
516            }
517            builder.append(QC);
518        }
519
520        builder.append("\"short_message\":\"");
521        final Message message = event.getMessage();
522        if (message instanceof CharSequence) {
523            JsonUtils.quoteAsString(((CharSequence) message), builder);
524        } else if (gcFree && message instanceof StringBuilderFormattable) {
525            final StringBuilder messageBuffer = getMessageStringBuilder();
526            try {
527                ((StringBuilderFormattable) message).formatTo(messageBuffer);
528                JsonUtils.quoteAsString(messageBuffer, builder);
529            } finally {
530                trimToMaxSize(messageBuffer);
531            }
532        } else {
533            JsonUtils.quoteAsString(toNullSafeString(message.getFormattedMessage()), builder);
534        }
535        builder.append(Q);
536        builder.append('}');
537        if (includeNullDelimiter) {
538            builder.append('\0');
539        }
540        if (includeNewLineDelimiter) {
541            builder.append('\n');
542        }
543        return builder;
544    }
545
546    private static boolean valueNeedsLookup(final String value) {
547        return value != null && value.contains("${");
548    }
549
550    private static class FieldWriter implements TriConsumer<String, Object, StringBuilder> {
551        private final ListChecker checker;
552
553        FieldWriter(ListChecker checker) {
554            this.checker = checker;
555        }
556
557        @Override
558        public void accept(final String key, final Object value, final StringBuilder stringBuilder) {
559            if (checker.check(key)) {
560                stringBuilder.append(QU);
561                JsonUtils.quoteAsString(key, stringBuilder);
562                stringBuilder.append("\":\"");
563                JsonUtils.quoteAsString(toNullSafeString(String.valueOf(value)), stringBuilder);
564                stringBuilder.append(QC);
565            }
566        }
567
568        public ListChecker getChecker() {
569            return checker;
570        }
571    }
572
573    private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<>();
574
575    private static StringBuilder getMessageStringBuilder() {
576        StringBuilder result = messageStringBuilder.get();
577        if (result == null) {
578            result = new StringBuilder(DEFAULT_STRING_BUILDER_SIZE);
579            messageStringBuilder.set(result);
580        }
581        result.setLength(0);
582        return result;
583    }
584
585    private static CharSequence toNullSafeString(final CharSequence s) {
586        return s == null ? Strings.EMPTY : s;
587    }
588
589    /**
590     * Non-private to make it accessible from unit test.
591     */
592    static CharSequence formatTimestamp(final long timeMillis) {
593        if (timeMillis < 1000) {
594            return "0";
595        }
596        final StringBuilder builder = getTimestampStringBuilder();
597        builder.append(timeMillis);
598        builder.insert(builder.length() - 3, '.');
599        return builder;
600    }
601
602    private static final ThreadLocal<StringBuilder> timestampStringBuilder = new ThreadLocal<>();
603
604    private static StringBuilder getTimestampStringBuilder() {
605        StringBuilder result = timestampStringBuilder.get();
606        if (result == null) {
607            result = new StringBuilder(20);
608            timestampStringBuilder.set(result);
609        }
610        result.setLength(0);
611        return result;
612    }
613
614    /**
615     * http://en.wikipedia.org/wiki/Syslog#Severity_levels
616     */
617    private int formatLevel(final Level level) {
618        return Severity.getSeverity(level).getCode();
619    }
620
621    /**
622     * Non-private to make it accessible from unit test.
623     */
624    static CharSequence formatThrowable(final Throwable throwable) {
625        // stack traces are big enough to provide a reasonably large initial capacity here
626        final StringWriter sw = new StringWriter(2048);
627        final PrintWriter pw = new PrintWriter(sw);
628        throwable.printStackTrace(pw);
629        pw.flush();
630        return sw.getBuffer();
631    }
632}