聊聊Elasticsearch的FsProbe

￡神魔★判官ぃ 2022-01-17 09:57 250阅读 0赞

[为什么80%的码农都做不了架构师？>>> ][80_]  ![hot3.png][]

## 序 ##

本文主要研究一下Elasticsearch的FsProbe

## FsProbe ##

elasticsearch-7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java

public class FsProbe {
    
        private static final Logger logger = LogManager.getLogger(FsProbe.class);
    
        private final NodeEnvironment nodeEnv;
    
        public FsProbe(NodeEnvironment nodeEnv) {
            this.nodeEnv = nodeEnv;
        }
    
        public FsInfo stats(FsInfo previous, @Nullable ClusterInfo clusterInfo) throws IOException {
            if (!nodeEnv.hasNodeFile()) {
                return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]);
            }
            NodePath[] dataLocations = nodeEnv.nodePaths();
            FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length];
            for (int i = 0; i < dataLocations.length; i++) {
                paths[i] = getFSInfo(dataLocations[i]);
            }
            FsInfo.IoStats ioStats = null;
            if (Constants.LINUX) {
                Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>();
                for (int i = 0; i < dataLocations.length; i++) {
                    if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) {
                        devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber));
                    }
                }
                ioStats = ioStats(devicesNumbers, previous);
            }
            DiskUsage leastDiskEstimate = null;
            DiskUsage mostDiskEstimate = null;
            if (clusterInfo != null) {
                leastDiskEstimate = clusterInfo.getNodeLeastAvailableDiskUsages().get(nodeEnv.nodeId());
                mostDiskEstimate = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeEnv.nodeId());
            }
            return new FsInfo(System.currentTimeMillis(), ioStats, paths, leastDiskEstimate, mostDiskEstimate);
        }
    
        final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers, final FsInfo previous) {
            try {
                final Map<Tuple<Integer, Integer>, FsInfo.DeviceStats> deviceMap = new HashMap<>();
                if (previous != null && previous.getIoStats() != null && previous.getIoStats().devicesStats != null) {
                    for (int i = 0; i < previous.getIoStats().devicesStats.length; i++) {
                        FsInfo.DeviceStats deviceStats = previous.getIoStats().devicesStats[i];
                        deviceMap.put(Tuple.tuple(deviceStats.majorDeviceNumber, deviceStats.minorDeviceNumber), deviceStats);
                    }
                }
    
                List<FsInfo.DeviceStats> devicesStats = new ArrayList<>();
    
                List<String> lines = readProcDiskStats();
                if (!lines.isEmpty()) {
                    for (String line : lines) {
                        String fields[] = line.trim().split("\\s+");
                        final int majorDeviceNumber = Integer.parseInt(fields[0]);
                        final int minorDeviceNumber = Integer.parseInt(fields[1]);
                        if (!devicesNumbers.contains(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))) {
                            continue;
                        }
                        final String deviceName = fields[2];
                        final long readsCompleted = Long.parseLong(fields[3]);
                        final long sectorsRead = Long.parseLong(fields[5]);
                        final long writesCompleted = Long.parseLong(fields[7]);
                        final long sectorsWritten = Long.parseLong(fields[9]);
                        final FsInfo.DeviceStats deviceStats =
                                new FsInfo.DeviceStats(
                                        majorDeviceNumber,
                                        minorDeviceNumber,
                                        deviceName,
                                        readsCompleted,
                                        sectorsRead,
                                        writesCompleted,
                                        sectorsWritten,
                                        deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber)));
                        devicesStats.add(deviceStats);
                    }
                }
    
                return new FsInfo.IoStats(devicesStats.toArray(new FsInfo.DeviceStats[devicesStats.size()]));
            } catch (Exception e) {
                // do not fail Elasticsearch if something unexpected
                // happens here
                logger.debug(() -> new ParameterizedMessage(
                        "unexpected exception processing /proc/diskstats for devices {}", devicesNumbers), e);
                return null;
            }
        }
    
        @SuppressForbidden(reason = "read /proc/diskstats")
        List<String> readProcDiskStats() throws IOException {
            return Files.readAllLines(PathUtils.get("/proc/diskstats"));
        }
    
        /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */
        /**
         * Take a large value intended to be positive, and if it has overflowed,
         * return {@code Long.MAX_VALUE} instead of a negative number.
         */
        static long adjustForHugeFilesystems(long bytes) {
            if (bytes < 0) {
                return Long.MAX_VALUE;
            }
            return bytes;
        }
    
        public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
            FsInfo.Path fsPath = new FsInfo.Path();
            fsPath.path = nodePath.path.toAbsolutePath().toString();
    
            // NOTE: we use already cached (on node startup) FileStore and spins
            // since recomputing these once per second (default) could be costly,
            // and they should not change:
            fsPath.total = adjustForHugeFilesystems(nodePath.fileStore.getTotalSpace());
            fsPath.free = adjustForHugeFilesystems(nodePath.fileStore.getUnallocatedSpace());
            fsPath.available = adjustForHugeFilesystems(nodePath.fileStore.getUsableSpace());
            fsPath.type = nodePath.fileStore.type();
            fsPath.mount = nodePath.fileStore.toString();
            return fsPath;
        }
    
    }

*  FsProbe提供了stats、ioStats、readProcDiskStats等方法；其中readProcDiskStats方法主要是读取`/proc/diskstats`的数据
 *  stats方法返回FsInfo，它包含了ioStats、leastDiskEstimate、mostDiskEstimate，其中ioStats是通过ioStats方法获取，而leastDiskEstimate及mostDiskEstimate则是通过clusterInfo.getNodeLeastAvailableDiskUsages()及clusterInfo.getNodeMostAvailableDiskUsages()获取
 *  ioStats方法则通过readProcDiskStats读取diskstats信息，构造FsInfo.DeviceStats，从而构造FsInfo.IoStats返回

## FsInfo ##

elasticsearch-7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContentFragment {
    	//......
    
        private final long timestamp;
        private final Path[] paths;
        private final IoStats ioStats;
        private final Path total;
        private final DiskUsage leastDiskEstimate;
        private final DiskUsage mostDiskEstimate;
    
        public FsInfo(long timestamp, IoStats ioStats, Path[] paths) {
            this(timestamp, ioStats, paths, null, null);
        }
    
        public FsInfo(long timestamp, IoStats ioStats, Path[] paths, @Nullable DiskUsage leastUsage, @Nullable DiskUsage mostUsage) {
            this.timestamp = timestamp;
            this.ioStats = ioStats;
            this.paths = paths;
            this.total = total();
            this.leastDiskEstimate = leastUsage;
            this.mostDiskEstimate = mostUsage;
        }
    
        /**
         * Read from a stream.
         */
        public FsInfo(StreamInput in) throws IOException {
            timestamp = in.readVLong();
            ioStats = in.readOptionalWriteable(IoStats::new);
            paths = new Path[in.readVInt()];
            for (int i = 0; i < paths.length; i++) {
                paths[i] = new Path(in);
            }
            this.total = total();
            if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
                this.leastDiskEstimate = in.readOptionalWriteable(DiskUsage::new);
                this.mostDiskEstimate = in.readOptionalWriteable(DiskUsage::new);
            } else {
                this.leastDiskEstimate = null;
                this.mostDiskEstimate = null;
            }
        }
    
        @Override
        public void writeTo(StreamOutput out) throws IOException {
            out.writeVLong(timestamp);
            out.writeOptionalWriteable(ioStats);
            out.writeVInt(paths.length);
            for (Path path : paths) {
                path.writeTo(out);
            }
            if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
                out.writeOptionalWriteable(this.leastDiskEstimate);
                out.writeOptionalWriteable(this.mostDiskEstimate);
            }
        }
    
        public Path getTotal() {
            return total;
        }
    
        @Nullable
        public DiskUsage getLeastDiskEstimate() {
            return this.leastDiskEstimate;
        }
    
        @Nullable
        public DiskUsage getMostDiskEstimate() {
            return this.mostDiskEstimate;
        }
    
        private Path total() {
            Path res = new Path();
            Set<String> seenDevices = new HashSet<>(paths.length);
            for (Path subPath : paths) {
                if (subPath.path != null) {
                    if (!seenDevices.add(subPath.path)) {
                        continue; // already added numbers for this device;
                    }
                }
                res.add(subPath);
            }
            return res;
        }
    
        public long getTimestamp() {
            return timestamp;
        }
    
        public IoStats getIoStats() {
            return ioStats;
        }
    
        @Override
        public Iterator<Path> iterator() {
            return Arrays.stream(paths).iterator();
        }
    
        @Override
        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
            builder.startObject(Fields.FS);
            builder.field(Fields.TIMESTAMP, timestamp);
            builder.field(Fields.TOTAL);
            total().toXContent(builder, params);
            if (leastDiskEstimate != null) {
                builder.startObject(Fields.LEAST_ESTIMATE);
                {
                    builder.field(Fields.PATH, leastDiskEstimate.getPath());
                    builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, new ByteSizeValue(leastDiskEstimate.getTotalBytes()));
                    builder.humanReadableField(Fields.AVAILABLE_IN_BYTES, Fields.AVAILABLE,
                        new ByteSizeValue(leastDiskEstimate.getFreeBytes()));
                    builder.field(Fields.USAGE_PERCENTAGE, leastDiskEstimate.getUsedDiskAsPercentage());
                }
                builder.endObject();
            }
    
            if (mostDiskEstimate != null) {
                builder.startObject(Fields.MOST_ESTIMATE);
                {
                    builder.field(Fields.PATH, mostDiskEstimate.getPath());
                    builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, new ByteSizeValue(mostDiskEstimate.getTotalBytes()));
                    builder.humanReadableField(Fields.AVAILABLE_IN_BYTES, Fields.AVAILABLE, new ByteSizeValue(mostDiskEstimate.getFreeBytes()));
                    builder.field(Fields.USAGE_PERCENTAGE, mostDiskEstimate.getUsedDiskAsPercentage());
                }
                builder.endObject();
            }
            builder.startArray(Fields.DATA);
            for (Path path : paths) {
                path.toXContent(builder, params);
            }
            builder.endArray();
            if (ioStats != null) {
                builder.startObject(Fields.IO_STATS);
                ioStats.toXContent(builder, params);
                builder.endObject();
            }
            builder.endObject();
            return builder;
        }
    
        static final class Fields {
            static final String FS = "fs";
            static final String TIMESTAMP = "timestamp";
            static final String DATA = "data";
            static final String TOTAL = "total";
            static final String TOTAL_IN_BYTES = "total_in_bytes";
            static final String IO_STATS = "io_stats";
            static final String LEAST_ESTIMATE = "least_usage_estimate";
            static final String MOST_ESTIMATE = "most_usage_estimate";
            static final String USAGE_PERCENTAGE = "used_disk_percent";
            static final String AVAILABLE = "available";
            static final String AVAILABLE_IN_BYTES = "available_in_bytes";
            static final String PATH = "path";
        }
    }

*  FsInfo定义了paths、ioStats、total、leastDiskEstimate、mostDiskEstimate属性，其内部定义了Path、DeviceStats、IoStats这三个静态类

### Path ###

elasticsearch-7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

public static class Path implements Writeable, ToXContentObject {
    
            String path;
            @Nullable
            String mount;
            /** File system type from {@code java.nio.file.FileStore type()}, if available. */
            @Nullable
            String type;
            long total = -1;
            long free = -1;
            long available = -1;
    
            public Path() {
            }
    
            public Path(String path, @Nullable String mount, long total, long free, long available) {
                this.path = path;
                this.mount = mount;
                this.total = total;
                this.free = free;
                this.available = available;
            }
    
            /**
             * Read from a stream.
             */
            public Path(StreamInput in) throws IOException {
                path = in.readOptionalString();
                mount = in.readOptionalString();
                type = in.readOptionalString();
                total = in.readLong();
                free = in.readLong();
                available = in.readLong();
                if (in.getVersion().before(Version.V_6_0_0_alpha1)) {
                    in.readOptionalBoolean();
                }
            }
    
            @Override
            public void writeTo(StreamOutput out) throws IOException {
                out.writeOptionalString(path); // total aggregates do not have a path
                out.writeOptionalString(mount);
                out.writeOptionalString(type);
                out.writeLong(total);
                out.writeLong(free);
                out.writeLong(available);
                if (out.getVersion().before(Version.V_6_0_0_alpha1)) {
                    out.writeOptionalBoolean(null);
                }
            }
    
            public String getPath() {
                return path;
            }
    
            public String getMount() {
                return mount;
            }
    
            public String getType() {
                return type;
            }
    
            public ByteSizeValue getTotal() {
                return new ByteSizeValue(total);
            }
    
            public ByteSizeValue getFree() {
                return new ByteSizeValue(free);
            }
    
            public ByteSizeValue getAvailable() {
                return new ByteSizeValue(available);
            }
    
            private long addLong(long current, long other) {
                if (other == -1) {
                    return current;
                }
                if (current == -1) {
                    return other;
                }
                return current + other;
            }
    
            private double addDouble(double current, double other) {
                if (other == -1) {
                    return current;
                }
                if (current == -1) {
                    return other;
                }
                return current + other;
            }
    
            public void add(Path path) {
                total = FsProbe.adjustForHugeFilesystems(addLong(total, path.total));
                free = FsProbe.adjustForHugeFilesystems(addLong(free, path.free));
                available = FsProbe.adjustForHugeFilesystems(addLong(available, path.available));
            }
    
            static final class Fields {
                static final String PATH = "path";
                static final String MOUNT = "mount";
                static final String TYPE = "type";
                static final String TOTAL = "total";
                static final String TOTAL_IN_BYTES = "total_in_bytes";
                static final String FREE = "free";
                static final String FREE_IN_BYTES = "free_in_bytes";
                static final String AVAILABLE = "available";
                static final String AVAILABLE_IN_BYTES = "available_in_bytes";
            }
    
            @Override
            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                builder.startObject();
                if (path != null) {
                    builder.field(Fields.PATH, path);
                }
                if (mount != null) {
                    builder.field(Fields.MOUNT, mount);
                }
                if (type != null) {
                    builder.field(Fields.TYPE, type);
                }
    
                if (total != -1) {
                    builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
                }
                if (free != -1) {
                    builder.humanReadableField(Fields.FREE_IN_BYTES, Fields.FREE, getFree());
                }
                if (available != -1) {
                    builder.humanReadableField(Fields.AVAILABLE_IN_BYTES, Fields.AVAILABLE, getAvailable());
                }
    
                builder.endObject();
                return builder;
            }
        }

*  Path定义了path、mount、type、total、free、available属性

### DeviceStats ###

elasticsearch-7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

public static class DeviceStats implements Writeable, ToXContentFragment {
    
            final int majorDeviceNumber;
            final int minorDeviceNumber;
            final String deviceName;
            final long currentReadsCompleted;
            final long previousReadsCompleted;
            final long currentSectorsRead;
            final long previousSectorsRead;
            final long currentWritesCompleted;
            final long previousWritesCompleted;
            final long currentSectorsWritten;
            final long previousSectorsWritten;
    
            public DeviceStats(
                    final int majorDeviceNumber,
                    final int minorDeviceNumber,
                    final String deviceName,
                    final long currentReadsCompleted,
                    final long currentSectorsRead,
                    final long currentWritesCompleted,
                    final long currentSectorsWritten,
                    final DeviceStats previousDeviceStats) {
                this(
                        majorDeviceNumber,
                        minorDeviceNumber,
                        deviceName,
                        currentReadsCompleted,
                        previousDeviceStats != null ? previousDeviceStats.currentReadsCompleted : -1,
                        currentSectorsWritten,
                        previousDeviceStats != null ? previousDeviceStats.currentSectorsWritten : -1,
                        currentSectorsRead,
                        previousDeviceStats != null ? previousDeviceStats.currentSectorsRead : -1,
                        currentWritesCompleted,
                        previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1);
            }
    
            private DeviceStats(
                    final int majorDeviceNumber,
                    final int minorDeviceNumber,
                    final String deviceName,
                    final long currentReadsCompleted,
                    final long previousReadsCompleted,
                    final long currentSectorsWritten,
                    final long previousSectorsWritten,
                    final long currentSectorsRead,
                    final long previousSectorsRead,
                    final long currentWritesCompleted,
                    final long previousWritesCompleted) {
                this.majorDeviceNumber = majorDeviceNumber;
                this.minorDeviceNumber = minorDeviceNumber;
                this.deviceName = deviceName;
                this.currentReadsCompleted = currentReadsCompleted;
                this.previousReadsCompleted = previousReadsCompleted;
                this.currentWritesCompleted = currentWritesCompleted;
                this.previousWritesCompleted = previousWritesCompleted;
                this.currentSectorsRead = currentSectorsRead;
                this.previousSectorsRead = previousSectorsRead;
                this.currentSectorsWritten = currentSectorsWritten;
                this.previousSectorsWritten = previousSectorsWritten;
            }
    
            public DeviceStats(StreamInput in) throws IOException {
                majorDeviceNumber = in.readVInt();
                minorDeviceNumber = in.readVInt();
                deviceName = in.readString();
                currentReadsCompleted = in.readLong();
                previousReadsCompleted = in.readLong();
                currentWritesCompleted = in.readLong();
                previousWritesCompleted = in.readLong();
                currentSectorsRead = in.readLong();
                previousSectorsRead = in.readLong();
                currentSectorsWritten = in.readLong();
                previousSectorsWritten = in.readLong();
            }
    
            @Override
            public void writeTo(StreamOutput out) throws IOException {
                out.writeVInt(majorDeviceNumber);
                out.writeVInt(minorDeviceNumber);
                out.writeString(deviceName);
                out.writeLong(currentReadsCompleted);
                out.writeLong(previousReadsCompleted);
                out.writeLong(currentWritesCompleted);
                out.writeLong(previousWritesCompleted);
                out.writeLong(currentSectorsRead);
                out.writeLong(previousSectorsRead);
                out.writeLong(currentSectorsWritten);
                out.writeLong(previousSectorsWritten);
            }
    
            public long operations() {
                if (previousReadsCompleted == -1 || previousWritesCompleted == -1) return -1;
    
                return (currentReadsCompleted - previousReadsCompleted) +
                    (currentWritesCompleted - previousWritesCompleted);
            }
    
            public long readOperations() {
                if (previousReadsCompleted == -1) return -1;
    
                return (currentReadsCompleted - previousReadsCompleted);
            }
    
            public long writeOperations() {
                if (previousWritesCompleted == -1) return -1;
    
                return (currentWritesCompleted - previousWritesCompleted);
            }
    
            public long readKilobytes() {
                if (previousSectorsRead == -1) return -1;
    
                return (currentSectorsRead - previousSectorsRead) / 2;
            }
    
            public long writeKilobytes() {
                if (previousSectorsWritten == -1) return -1;
    
                return (currentSectorsWritten - previousSectorsWritten) / 2;
            }
    
            @Override
            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                builder.field("device_name", deviceName);
                builder.field(IoStats.OPERATIONS, operations());
                builder.field(IoStats.READ_OPERATIONS, readOperations());
                builder.field(IoStats.WRITE_OPERATIONS, writeOperations());
                builder.field(IoStats.READ_KILOBYTES, readKilobytes());
                builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes());
                return builder;
            }
    
        }

*  DeviceStats定义了majorDeviceNumber、minorDeviceNumber、deviceName、currentReadsCompleted、currentSectorsRead、currentWritesCompleted、currentSectorsWritten、previousDeviceStats属性

### IoStats ###

elasticsearch-7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

public static class IoStats implements Writeable, ToXContentFragment {
    
            private static final String OPERATIONS = "operations";
            private static final String READ_OPERATIONS = "read_operations";
            private static final String WRITE_OPERATIONS = "write_operations";
            private static final String READ_KILOBYTES = "read_kilobytes";
            private static final String WRITE_KILOBYTES = "write_kilobytes";
    
            final DeviceStats[] devicesStats;
            final long totalOperations;
            final long totalReadOperations;
            final long totalWriteOperations;
            final long totalReadKilobytes;
            final long totalWriteKilobytes;
    
            public IoStats(final DeviceStats[] devicesStats) {
                this.devicesStats = devicesStats;
                long totalOperations = 0;
                long totalReadOperations = 0;
                long totalWriteOperations = 0;
                long totalReadKilobytes = 0;
                long totalWriteKilobytes = 0;
                for (DeviceStats deviceStats : devicesStats) {
                    totalOperations += deviceStats.operations() != -1 ? deviceStats.operations() : 0;
                    totalReadOperations += deviceStats.readOperations() != -1 ? deviceStats.readOperations() : 0;
                    totalWriteOperations += deviceStats.writeOperations() != -1 ? deviceStats.writeOperations() : 0;
                    totalReadKilobytes += deviceStats.readKilobytes() != -1 ? deviceStats.readKilobytes() : 0;
                    totalWriteKilobytes += deviceStats.writeKilobytes() != -1 ? deviceStats.writeKilobytes() : 0;
                }
                this.totalOperations = totalOperations;
                this.totalReadOperations = totalReadOperations;
                this.totalWriteOperations = totalWriteOperations;
                this.totalReadKilobytes = totalReadKilobytes;
                this.totalWriteKilobytes = totalWriteKilobytes;
            }
    
            public IoStats(StreamInput in) throws IOException {
                final int length = in.readVInt();
                final DeviceStats[] devicesStats = new DeviceStats[length];
                for (int i = 0; i < length; i++) {
                    devicesStats[i] = new DeviceStats(in);
                }
                this.devicesStats = devicesStats;
                this.totalOperations = in.readLong();
                this.totalReadOperations = in.readLong();
                this.totalWriteOperations = in.readLong();
                this.totalReadKilobytes = in.readLong();
                this.totalWriteKilobytes = in.readLong();
            }
    
            @Override
            public void writeTo(StreamOutput out) throws IOException {
                out.writeVInt(devicesStats.length);
                for (int i = 0; i < devicesStats.length; i++) {
                    devicesStats[i].writeTo(out);
                }
                out.writeLong(totalOperations);
                out.writeLong(totalReadOperations);
                out.writeLong(totalWriteOperations);
                out.writeLong(totalReadKilobytes);
                out.writeLong(totalWriteKilobytes);
            }
    
            public DeviceStats[] getDevicesStats() {
                return devicesStats;
            }
    
            public long getTotalOperations() {
                return totalOperations;
            }
    
            public long getTotalReadOperations() {
                return totalReadOperations;
            }
    
            public long getTotalWriteOperations() {
                return totalWriteOperations;
            }
    
            public long getTotalReadKilobytes() {
                return totalReadKilobytes;
            }
    
            public long getTotalWriteKilobytes() {
                return totalWriteKilobytes;
            }
    
            @Override
            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                if (devicesStats.length > 0) {
                    builder.startArray("devices");
                    for (DeviceStats deviceStats : devicesStats) {
                        builder.startObject();
                        deviceStats.toXContent(builder, params);
                        builder.endObject();
                    }
                    builder.endArray();
    
                    builder.startObject("total");
                    builder.field(OPERATIONS, totalOperations);
                    builder.field(READ_OPERATIONS, totalReadOperations);
                    builder.field(WRITE_OPERATIONS, totalWriteOperations);
                    builder.field(READ_KILOBYTES, totalReadKilobytes);
                    builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
                    builder.endObject();
                }
                return builder;
            }
    
        }

*  IoStats定义了devicesStats、totalOperations、totalReadOperations、totalWriteOperations、totalReadKilobytes、totalWriteKilobytes属性

## 小结 ##

## doc ##

*  [FsProbe][]

转载于:https://my.oschina.net/go4it/blog/3053852

[80_]: https://my.oschina.net/u/2663968/blog/3051541
[hot3.png]: /images/20220117/a8914b4f7cd34a13b162a7bdbb1faada.png
[FsProbe]: https://github.com/elastic/elasticsearch/blob/v7.0.1/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java