arthas-master

core/src/main/java/com/taobao/arthas/core/command/monitor200/JvmCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.model.JvmModel;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import java.lang.management.*;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * JVM info command
+ *
+ * @author vlinux on 15/6/6.
+ */
+@Name("jvm")
+@Summary("Display the target JVM information")
+@Description(Constants.WIKI + Constants.WIKI_HOME + "jvm")
+public class JvmCommand extends AnnotatedCommand {
+
+    private final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+    private final ClassLoadingMXBean classLoadingMXBean = ManagementFactory.getClassLoadingMXBean();
+    private final CompilationMXBean compilationMXBean = ManagementFactory.getCompilationMXBean();
+    private final Collection<GarbageCollectorMXBean> garbageCollectorMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
+    private final Collection<MemoryManagerMXBean> memoryManagerMXBeans = ManagementFactory.getMemoryManagerMXBeans();
+    private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+    //    private final Collection<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+    private final OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean();
+    private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+
+    @Override
+    public void process(CommandProcess process) {
+
+        JvmModel jvmModel = new JvmModel();
+
+        addRuntimeInfo(jvmModel);
+
+        addClassLoading(jvmModel);
+
+        addCompilation(jvmModel);
+
+        if (!garbageCollectorMXBeans.isEmpty()) {
+            addGarbageCollectors(jvmModel);
+        }
+
+        if (!memoryManagerMXBeans.isEmpty()) {
+            addMemoryManagers(jvmModel);
+        }
+
+        addMemory(jvmModel);
+
+        addOperatingSystem(jvmModel);
+
+        addThread(jvmModel);
+
+        addFileDescriptor(jvmModel);
+
+        process.appendResult(jvmModel);
+        process.end();
+    }
+
+    private void addFileDescriptor(JvmModel jvmModel) {
+        String group = "FILE-DESCRIPTOR";
+        jvmModel.addItem(group,"MAX-FILE-DESCRIPTOR-COUNT", invokeFileDescriptor(operatingSystemMXBean, "getMaxFileDescriptorCount"))
+                .addItem(group,"OPEN-FILE-DESCRIPTOR-COUNT", invokeFileDescriptor(operatingSystemMXBean, "getOpenFileDescriptorCount"));
+    }
+
+    private long invokeFileDescriptor(OperatingSystemMXBean os, String name) {
+        try {
+            final Method method = os.getClass().getDeclaredMethod(name);
+            method.setAccessible(true);
+            return (Long) method.invoke(os);
+        } catch (Exception e) {
+            return -1;
+        }
+    }
+
+    private void addRuntimeInfo(JvmModel jvmModel) {
+        String bootClassPath = "";
+        try {
+            bootClassPath = runtimeMXBean.getBootClassPath();
+        } catch (Exception e) {
+            // under jdk9 will throw UnsupportedOperationException, ignore
+        }
+        String group = "RUNTIME";
+        jvmModel.addItem(group,"MACHINE-NAME", runtimeMXBean.getName());
+        jvmModel.addItem(group, "JVM-START-TIME", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(runtimeMXBean.getStartTime())));
+        jvmModel.addItem(group, "MANAGEMENT-SPEC-VERSION", runtimeMXBean.getManagementSpecVersion());
+        jvmModel.addItem(group, "SPEC-NAME", runtimeMXBean.getSpecName());
+        jvmModel.addItem(group, "SPEC-VENDOR", runtimeMXBean.getSpecVendor());
+        jvmModel.addItem(group, "SPEC-VERSION", runtimeMXBean.getSpecVersion());
+        jvmModel.addItem(group, "VM-NAME", runtimeMXBean.getVmName());
+        jvmModel.addItem(group, "VM-VENDOR", runtimeMXBean.getVmVendor());
+        jvmModel.addItem(group, "VM-VERSION", runtimeMXBean.getVmVersion());
+        jvmModel.addItem(group, "INPUT-ARGUMENTS", runtimeMXBean.getInputArguments());
+        jvmModel.addItem(group, "CLASS-PATH", runtimeMXBean.getClassPath());
+        jvmModel.addItem(group, "BOOT-CLASS-PATH", bootClassPath);
+        jvmModel.addItem(group, "LIBRARY-PATH", runtimeMXBean.getLibraryPath());
+    }
+
+    private void addClassLoading(JvmModel jvmModel) {
+        String group = "CLASS-LOADING";
+        jvmModel.addItem(group, "LOADED-CLASS-COUNT", classLoadingMXBean.getLoadedClassCount());
+        jvmModel.addItem(group, "TOTAL-LOADED-CLASS-COUNT", classLoadingMXBean.getTotalLoadedClassCount());
+        jvmModel.addItem(group, "UNLOADED-CLASS-COUNT", classLoadingMXBean.getUnloadedClassCount());
+        jvmModel.addItem(group, "IS-VERBOSE", classLoadingMXBean.isVerbose());
+    }
+
+    private void addCompilation(JvmModel jvmModel) {
+        if (compilationMXBean == null) {
+            return;
+        }
+        String group = "COMPILATION";
+        jvmModel.addItem(group, "NAME", compilationMXBean.getName());
+        if (compilationMXBean.isCompilationTimeMonitoringSupported()) {
+            jvmModel.addItem(group, "TOTAL-COMPILE-TIME", compilationMXBean.getTotalCompilationTime(), "time (ms)");
+        }
+    }
+
+    private void addGarbageCollectors(JvmModel jvmModel) {
+        String group = "GARBAGE-COLLECTORS";
+        for (GarbageCollectorMXBean gcMXBean : garbageCollectorMXBeans) {
+            Map<String, Object> gcInfo = new LinkedHashMap<String, Object>();
+            gcInfo.put("name", gcMXBean.getName());
+            gcInfo.put("collectionCount", gcMXBean.getCollectionCount());
+            gcInfo.put("collectionTime", gcMXBean.getCollectionTime());
+
+            jvmModel.addItem(group, gcMXBean.getName(), gcInfo, "count/time (ms)");
+        }
+    }
+
+    private void addMemoryManagers(JvmModel jvmModel) {
+        String group = "MEMORY-MANAGERS";
+        for (final MemoryManagerMXBean memoryManagerMXBean : memoryManagerMXBeans) {
+            if (memoryManagerMXBean.isValid()) {
+                final String name = memoryManagerMXBean.isValid()
+                        ? memoryManagerMXBean.getName()
+                        : memoryManagerMXBean.getName() + "(Invalid)";
+                jvmModel.addItem(group, name, memoryManagerMXBean.getMemoryPoolNames());
+            }
+        }
+    }
+
+    private void addMemory(JvmModel jvmModel) {
+        String group = "MEMORY";
+        MemoryUsage heapMemoryUsage = memoryMXBean.getHeapMemoryUsage();
+        Map<String, Object> heapMemoryInfo = getMemoryUsageInfo("heap", heapMemoryUsage);
+        jvmModel.addItem(group, "HEAP-MEMORY-USAGE", heapMemoryInfo, "memory in bytes");
+
+        MemoryUsage nonHeapMemoryUsage = memoryMXBean.getNonHeapMemoryUsage();
+        Map<String, Object> nonheapMemoryInfo = getMemoryUsageInfo("nonheap", nonHeapMemoryUsage);
+        jvmModel.addItem(group,"NO-HEAP-MEMORY-USAGE", nonheapMemoryInfo, "memory in bytes");
+
+        jvmModel.addItem(group,"PENDING-FINALIZE-COUNT", memoryMXBean.getObjectPendingFinalizationCount());
+    }
+
+    private Map<String, Object> getMemoryUsageInfo(String name, MemoryUsage heapMemoryUsage) {
+        Map<String, Object> memoryInfo = new LinkedHashMap<String, Object>();
+        memoryInfo.put("name", name);
+        memoryInfo.put("init", heapMemoryUsage.getInit());
+        memoryInfo.put("used", heapMemoryUsage.getUsed());
+        memoryInfo.put("committed", heapMemoryUsage.getCommitted());
+        memoryInfo.put("max", heapMemoryUsage.getMax());
+        return memoryInfo;
+    }
+
+    private void addOperatingSystem(JvmModel jvmModel) {
+        String group = "OPERATING-SYSTEM";
+        jvmModel.addItem(group,"OS", operatingSystemMXBean.getName())
+                .addItem(group,"ARCH", operatingSystemMXBean.getArch())
+                .addItem(group,"PROCESSORS-COUNT", operatingSystemMXBean.getAvailableProcessors())
+                .addItem(group,"LOAD-AVERAGE", operatingSystemMXBean.getSystemLoadAverage())
+                .addItem(group,"VERSION", operatingSystemMXBean.getVersion());
+    }
+
+    private void addThread(JvmModel jvmModel) {
+        String group = "THREAD";
+        jvmModel.addItem(group, "COUNT", threadMXBean.getThreadCount())
+                .addItem(group, "DAEMON-COUNT", threadMXBean.getDaemonThreadCount())
+                .addItem(group, "PEAK-COUNT", threadMXBean.getPeakThreadCount())
+                .addItem(group, "STARTED-COUNT", threadMXBean.getTotalStartedThreadCount())
+                .addItem(group, "DEADLOCK-COUNT",getDeadlockedThreadsCount(threadMXBean));
+    }
+
+    private int getDeadlockedThreadsCount(ThreadMXBean threads) {
+        final long[] ids = threads.findDeadlockedThreads();
+        if (ids == null) {
+            return 0;
+        } else {
+            return ids.length;
+        }
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/MBeanCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.model.MBeanAttributeVO;
+import com.taobao.arthas.core.command.model.MBeanModel;
+import com.taobao.arthas.core.shell.cli.CliToken;
+import com.taobao.arthas.core.shell.cli.Completion;
+import com.taobao.arthas.core.shell.cli.CompletionUtils;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.shell.handlers.Handler;
+import com.taobao.arthas.core.shell.handlers.command.CommandInterruptHandler;
+import com.taobao.arthas.core.shell.handlers.shell.QExitHandler;
+import com.taobao.arthas.core.shell.session.Session;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.arthas.core.util.TokenUtils;
+import com.taobao.arthas.core.util.matcher.Matcher;
+import com.taobao.arthas.core.util.matcher.RegexMatcher;
+import com.taobao.arthas.core.util.matcher.WildcardMatcher;
+import com.taobao.middleware.cli.annotations.Argument;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.TabularData;
+
+/**
+ * Date: 2019/4/18
+ *
+ * @author xuzhiyi
+ */
+@Name("mbean")
+@Summary("Display the mbean information")
+@Description("\nExamples:\n" +
+        "  mbean\n" +
+        "  mbean -m java.lang:type=Threading\n" +
+        "  mbean java.lang:type=Threading\n" +
+        "  mbean java.lang:type=Threading *Count\n" +
+        "  mbean java.lang:type=MemoryPool,name=PS\\ Old\\ Gen\n" +
+        "  mbean java.lang:type=MemoryPool,name=*\n" +
+        "  mbean java.lang:type=MemoryPool,name=* Usage\n" +
+        "  mbean -E java.lang:type=Threading PeakThreadCount|ThreadCount|DaemonThreadCount\n" +
+        "  mbean -i 1000 java.lang:type=Threading *Count\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "mbean")
+public class MBeanCommand extends AnnotatedCommand {
+
+    private static final Logger logger = LoggerFactory.getLogger(MBeanCommand.class);
+
+    private String name;
+    private String attribute;
+    private boolean isRegEx = false;
+    private long interval = 0;
+    private boolean metaData;
+    private int numOfExecutions = 100;
+    private Timer timer;
+    private long count = 0;
+
+    @Argument(argName = "name-pattern", index = 0, required = false)
+    @Description("ObjectName pattern, see javax.management.ObjectName for more detail. \n" +
+            "It looks like this: \n" +
+            "  domain: key-property-list\n" +
+            "For example: \n" +
+            "  java.lang:name=G1 Old Gen,type=MemoryPool\n" +
+            "  java.lang:name=*,type=MemoryPool")
+    public void setNamePattern(String name) {
+        this.name = name;
+    }
+
+    @Argument(argName = "attribute-pattern", index = 1, required = false)
+    @Description("Attribute name pattern.")
+    public void setAttributePattern(String attribute) {
+        this.attribute = attribute;
+    }
+
+    @Option(shortName = "i", longName = "interval")
+    @Description("The interval (in ms) between two executions.")
+    public void setInterval(long interval) {
+        this.interval = interval;
+    }
+
+    @Option(shortName = "E", longName = "regex", flag = true)
+    @Description("Enable regular expression to match attribute name (wildcard matching by default).")
+    public void setRegEx(boolean regEx) {
+        isRegEx = regEx;
+    }
+
+    @Option(shortName = "m", longName = "metadata", flag = true)
+    @Description("Show metadata of mbean.")
+    public void setMetaData(boolean metaData) {
+        this.metaData = metaData;
+    }
+
+    @Option(shortName = "n", longName = "number-of-execution")
+    @Description("The number of times this command will be executed.")
+    public void setNumOfExecutions(int numOfExecutions) {
+        this.numOfExecutions = numOfExecutions;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public boolean isRegEx() {
+        return isRegEx;
+    }
+
+    public boolean isMetaData() {
+        return metaData;
+    }
+
+    public long getInterval() {
+        return interval;
+    }
+
+    public int getNumOfExecutions() {
+        return numOfExecutions;
+    }
+
+    @Override
+    public void process(CommandProcess process) {
+        //每个分支调用process.end()结束执行
+        if (StringUtils.isEmpty(getName())) {
+            listMBean(process);
+        } else if (isMetaData()) {
+            listMetaData(process);
+        } else {
+            listAttribute(process);
+        }
+    }
+
+    private void listMBean(CommandProcess process) {
+        Set<ObjectName> objectNames = queryObjectNames();
+        List<String> mbeanNames = new ArrayList<String>(objectNames.size());
+        for (ObjectName objectName : objectNames) {
+            mbeanNames.add(objectName.toString());
+        }
+        process.appendResult(new MBeanModel(mbeanNames));
+        process.end();
+    }
+
+    private void listAttribute(final CommandProcess process) {
+        Session session = process.session();
+        timer = new Timer("Timer-for-arthas-mbean-" + session.getSessionId(), true);
+
+        // ctrl-C support
+        process.interruptHandler(new MBeanInterruptHandler(process, timer));
+
+        // 通过handle回调，在suspend和end时停止timer，resume时重启timer
+        Handler<Void> stopHandler = new Handler<Void>() {
+            @Override
+            public void handle(Void event) {
+                stop();
+            }
+        };
+
+        Handler<Void> restartHandler = new Handler<Void>() {
+            @Override
+            public void handle(Void event) {
+                restart(process);
+            }
+        };
+        process.suspendHandler(stopHandler);
+        process.resumeHandler(restartHandler);
+        process.endHandler(stopHandler);
+        // q exit support
+        process.stdinHandler(new QExitHandler(process));
+
+        // start the timer
+        if (getInterval() > 0) {
+            timer.scheduleAtFixedRate(new MBeanTimerTask(process), 0, getInterval());
+        } else {
+            timer.schedule(new MBeanTimerTask(process), 0);
+        }
+
+        //异步执行，这里不能调用process.end()，在timer task中结束命令执行
+    }
+
+    public synchronized void stop() {
+        if (timer != null) {
+            timer.cancel();
+            timer.purge();
+            timer = null;
+        }
+    }
+
+    public synchronized void restart(CommandProcess process) {
+        if (timer == null) {
+            Session session = process.session();
+            timer = new Timer("Timer-for-arthas-mbean-" + session.getSessionId(), true);
+            timer.scheduleAtFixedRate(new MBeanTimerTask(process), 0, getInterval());
+        }
+    }
+
+    private void listMetaData(CommandProcess process) {
+        Set<ObjectName> objectNames = queryObjectNames();
+        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
+        try {
+            MBeanModel mbeanModel = new MBeanModel();
+            Map<String, MBeanInfo> mbeanMetaData = new LinkedHashMap<String, MBeanInfo>();
+            mbeanModel.setMbeanMetadata(mbeanMetaData);
+            for (ObjectName objectName : objectNames) {
+                MBeanInfo mBeanInfo = mBeanServer.getMBeanInfo(objectName);
+                mbeanMetaData.put(objectName.toString(), mBeanInfo);
+            }
+            process.appendResult(mbeanModel);
+            process.end();
+        } catch (Throwable e) {
+            logger.warn("listMetaData error", e);
+            process.end(1, "list mbean metadata error");
+        }
+    }
+
+    @Override
+    public void complete(Completion completion) {
+        int argumentIndex = CompletionUtils.detectArgumentIndex(completion);
+        if (argumentIndex == 1) {
+            if (!completeBeanName(completion)) {
+                super.complete(completion);
+            }
+            return;
+        } else if (argumentIndex == 2) {
+            if (!completeAttributeName(completion)) {
+                super.complete(completion);
+            }
+            return;
+        }
+        super.complete(completion);
+    }
+
+    private boolean completeBeanName(Completion completion) {
+        List<CliToken> tokens = completion.lineTokens();
+        String lastToken = TokenUtils.getLast(tokens).value();
+
+        if (StringUtils.isBlank(lastToken)) {
+            lastToken = "";
+        }
+
+        if (lastToken.startsWith("-") || lastToken.startsWith("--")) {
+            return false;
+        }
+
+        Set<ObjectName> objectNames = queryObjectNames();
+        Set<String> names = new HashSet<String>();
+
+        if (objectNames == null) {
+            return false;
+        }
+        for (ObjectName objectName : objectNames) {
+            String name = objectName.toString();
+            if (name.startsWith(lastToken)) {
+                int index = name.indexOf('.', lastToken.length());
+                if (index > 0) {
+                    names.add(name.substring(0, index + 1));
+                    continue;
+                }
+                index = name.indexOf(':', lastToken.length());
+                if (index > 0) {
+                    names.add(name.substring(0, index + 1));
+                    continue;
+                }
+                names.add(name);
+            }
+        }
+        String next = names.iterator().next();
+        if (names.size() == 1 && (next.endsWith(".") || next.endsWith(":"))) {
+            completion.complete(next.substring(lastToken.length()), false);
+            return true;
+        } else {
+            return CompletionUtils.complete(completion, names);
+        }
+    }
+
+    private boolean completeAttributeName(Completion completion) {
+        List<CliToken> tokens = completion.lineTokens();
+        String lastToken = TokenUtils.getLast(tokens).value();
+
+        if (StringUtils.isBlank(lastToken)) {
+            lastToken = "";
+        }
+
+        MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+        String beanName = TokenUtils.retrievePreviousArg(tokens, lastToken);
+        Set<ObjectName> objectNames = null;
+        try {
+            objectNames = platformMBeanServer.queryNames(new ObjectName(beanName), null);
+        } catch (MalformedObjectNameException e) {
+            logger.warn("queryNames error", e);
+        }
+        if (objectNames == null || objectNames.size() == 0) {
+            return false;
+        }
+        try {
+            MBeanInfo mBeanInfo = platformMBeanServer.getMBeanInfo(objectNames.iterator().next());
+            List<String> attributeNames = new ArrayList<String>();
+            MBeanAttributeInfo[] attributes = mBeanInfo.getAttributes();
+            for (MBeanAttributeInfo attribute : attributes) {
+                if (StringUtils.isBlank(lastToken)) {
+                    attributeNames.add(attribute.getName());
+                } else if (attribute.getName().startsWith(lastToken)) {
+                    attributeNames.add(attribute.getName());
+                }
+            }
+            return CompletionUtils.complete(completion, attributeNames);
+        } catch (Throwable e) {
+            logger.warn("getMBeanInfo error", e);
+        }
+        return false;
+    }
+
+    private Set<ObjectName> queryObjectNames() {
+        MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+        Set<ObjectName> objectNames = new HashSet<ObjectName>();
+        try {
+            if (StringUtils.isEmpty(name)) {
+                name = "*:*";
+            }
+            objectNames = platformMBeanServer.queryNames(new ObjectName(name), null);
+        } catch (MalformedObjectNameException e) {
+            logger.warn("queryObjectNames error", e);
+        }
+        return objectNames;
+    }
+
+    private Matcher<String> getAttributeMatcher() {
+        if (StringUtils.isEmpty(attribute)) {
+            attribute = isRegEx ? ".*" : "*";
+        }
+        return isRegEx ? new RegexMatcher(attribute) : new WildcardMatcher(attribute);
+    }
+
+
+    public static class MBeanInterruptHandler extends CommandInterruptHandler {
+
+        private volatile Timer timer;
+
+        public MBeanInterruptHandler(CommandProcess process, Timer timer) {
+            super(process);
+            this.timer = timer;
+        }
+
+        @Override
+        public void handle(Void event) {
+            timer.cancel();
+            super.handle(event);
+        }
+    }
+
+    private class MBeanTimerTask extends TimerTask {
+
+        private CommandProcess process;
+
+        public MBeanTimerTask(CommandProcess process) {
+            this.process = process;
+        }
+
+        @Override
+        public void run() {
+            if (count >= getNumOfExecutions()) {
+                // stop the timer
+                timer.cancel();
+                timer.purge();
+                process.end(-1, "Process ends after " + getNumOfExecutions() + " time(s).");
+                return;
+            }
+
+            try {
+                //result model
+                MBeanModel mBeanModel = new MBeanModel();
+                Map<String, List<MBeanAttributeVO>> mbeanAttributeMap = new LinkedHashMap<String, List<MBeanAttributeVO>>();
+                mBeanModel.setMbeanAttribute(mbeanAttributeMap);
+
+                MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+                Set<ObjectName> objectNames = queryObjectNames();
+                for (ObjectName objectName : objectNames) {
+                    List<MBeanAttributeVO> attributeVOs = null;
+                    MBeanInfo mBeanInfo = platformMBeanServer.getMBeanInfo(objectName);
+                    MBeanAttributeInfo[] attributes = mBeanInfo.getAttributes();
+                    for (MBeanAttributeInfo attribute : attributes) {
+                        String attributeName = attribute.getName();
+                        if (!getAttributeMatcher().matching(attributeName)) {
+                            continue;
+                        }
+
+                        //create attributeVO list
+                        if (attributeVOs == null) {
+                            attributeVOs = new ArrayList<MBeanAttributeVO>();
+                            mbeanAttributeMap.put(objectName.toString(), attributeVOs);
+                        }
+
+                        if (!attribute.isReadable()) {
+                            attributeVOs.add(new MBeanAttributeVO(attributeName, null, "Unavailable"));
+                        } else {
+                            try {
+                                Object attributeObj = platformMBeanServer.getAttribute(objectName, attributeName);
+                                attributeVOs.add(createMBeanAttributeVO(attributeName, attributeObj));
+                            } catch (Throwable e) {
+                                logger.error("read mbean attribute failed: objectName={}, attributeName={}", objectName, attributeName, e);
+                                String errorStr;
+                                Throwable cause = e.getCause();
+                                if (cause instanceof UnsupportedOperationException) {
+                                    errorStr = "Unsupported";
+                                } else {
+                                    errorStr = "Failure";
+                                }
+                                attributeVOs.add(new MBeanAttributeVO(attributeName, null, errorStr));
+                            }
+                        }
+                    }
+                }
+                process.appendResult(mBeanModel);
+            } catch (Throwable e) {
+                logger.warn("read mbean error", e);
+                stop();
+                process.end(1, "read mbean error.");
+                return;
+            }
+
+            count++;
+            process.times().incrementAndGet();
+            if (getInterval() <= 0) {
+                stop();
+                process.end();
+            }
+        }
+    }
+
+    private MBeanAttributeVO createMBeanAttributeVO(String attributeName, Object originAttrValue) {
+        Object attrValue = convertAttrValue(attributeName, originAttrValue);
+
+        return new MBeanAttributeVO(attributeName, attrValue);
+    }
+
+    private Object convertAttrValue(String attributeName, Object originAttrValue) {
+        Object attrValue = originAttrValue;
+
+        try {
+            if (originAttrValue instanceof ObjectName) {
+                attrValue = String.valueOf(originAttrValue);
+            } else if (attrValue instanceof CompositeData) {
+                //mbean java.lang:type=MemoryPool,name=*
+                CompositeData compositeData = (CompositeData) attrValue;
+                attrValue = convertCompositeData(attributeName, compositeData);
+            } else if (attrValue instanceof CompositeData[]) {
+                //mbean com.sun.management:type=HotSpotDiagnostic
+                CompositeData[] compositeDataArray = (CompositeData[]) attrValue;
+                List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>(compositeDataArray.length);
+                for (CompositeData compositeData : compositeDataArray) {
+                    dataList.add(convertCompositeData(attributeName, compositeData));
+                }
+                attrValue = dataList;
+            } else if (attrValue instanceof TabularData) {
+                //mbean java.lang:type=GarbageCollector,name=*
+                TabularData tabularData = (TabularData) attrValue;
+                Collection<CompositeData> compositeDataList = (Collection<CompositeData>) tabularData.values();
+                List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>(compositeDataList.size());
+                for (CompositeData compositeData : compositeDataList) {
+                    dataList.add(convertCompositeData(attributeName, compositeData));
+                }
+                attrValue = dataList;
+            }
+        } catch (Throwable e) {
+            logger.error("convert mbean attribute error, attribute: {}={}", attributeName, originAttrValue, e);
+            attrValue = String.valueOf(originAttrValue);
+        }
+        return attrValue;
+    }
+
+    private Map<String, Object> convertCompositeData(String attributeName, CompositeData compositeData) {
+        Set<String> keySet = compositeData.getCompositeType().keySet();
+        String[] keys = keySet.toArray(new String[0]);
+        Object[] values = compositeData.getAll(keys);
+        Map<String, Object> data = new LinkedHashMap<String, Object>();
+        for (int i = 0; i < keys.length; i++) {
+            data.put(keys[i], convertAttrValue(attributeName + "." + keys[i], values[i]));
+        }
+        return data;
+    }
+}
\ No newline at end of file

core/src/main/java/com/taobao/arthas/core/command/monitor200/MemoryCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import static com.taobao.arthas.core.command.model.MemoryEntryVO.TYPE_BUFFER_POOL;
+import static com.taobao.arthas.core.command.model.MemoryEntryVO.TYPE_HEAP;
+import static com.taobao.arthas.core.command.model.MemoryEntryVO.TYPE_NON_HEAP;
+
+import java.lang.management.BufferPoolMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryType;
+import java.lang.management.MemoryUsage;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.model.MemoryEntryVO;
+import com.taobao.arthas.core.command.model.MemoryModel;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Summary;
+
+/**
+ * @author hengyunabc 2022-03-01
+ */
+@Name("memory")
+@Summary("Display jvm memory info.")
+@Description(Constants.EXAMPLE + "  memory\n" + Constants.WIKI + Constants.WIKI_HOME + "memory")
+public class MemoryCommand extends AnnotatedCommand {
+    @Override
+    public void process(CommandProcess process) {
+        MemoryModel result = new MemoryModel();
+        result.setMemoryInfo(memoryInfo());
+        process.appendResult(result);
+        process.end();
+    }
+
+    static Map<String, List<MemoryEntryVO>> memoryInfo() {
+        List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+        Map<String, List<MemoryEntryVO>> memoryInfoMap = new LinkedHashMap<String, List<MemoryEntryVO>>();
+
+        // heap
+        MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+        List<MemoryEntryVO> heapMemEntries = new ArrayList<MemoryEntryVO>();
+        heapMemEntries.add(createMemoryEntryVO(TYPE_HEAP, TYPE_HEAP, heapMemoryUsage));
+        for (MemoryPoolMXBean poolMXBean : memoryPoolMXBeans) {
+            if (MemoryType.HEAP.equals(poolMXBean.getType())) {
+                MemoryUsage usage = getUsage(poolMXBean);
+                if (usage != null) {
+                    String poolName = StringUtils.beautifyName(poolMXBean.getName());
+                    heapMemEntries.add(createMemoryEntryVO(TYPE_HEAP, poolName, usage));
+                }
+            }
+        }
+        memoryInfoMap.put(TYPE_HEAP, heapMemEntries);
+
+        // non-heap
+        MemoryUsage nonHeapMemoryUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage();
+        List<MemoryEntryVO> nonheapMemEntries = new ArrayList<MemoryEntryVO>();
+        nonheapMemEntries.add(createMemoryEntryVO(TYPE_NON_HEAP, TYPE_NON_HEAP, nonHeapMemoryUsage));
+        for (MemoryPoolMXBean poolMXBean : memoryPoolMXBeans) {
+            if (MemoryType.NON_HEAP.equals(poolMXBean.getType())) {
+                MemoryUsage usage = getUsage(poolMXBean);
+                if (usage != null) {
+                    String poolName = StringUtils.beautifyName(poolMXBean.getName());
+                    nonheapMemEntries.add(createMemoryEntryVO(TYPE_NON_HEAP, poolName, usage));
+                }
+            }
+        }
+        memoryInfoMap.put(TYPE_NON_HEAP, nonheapMemEntries);
+
+        addBufferPoolMemoryInfo(memoryInfoMap);
+        return memoryInfoMap;
+    }
+
+    private static MemoryUsage getUsage(MemoryPoolMXBean memoryPoolMXBean) {
+        try {
+            return memoryPoolMXBean.getUsage();
+        } catch (InternalError e) {
+            // Defensive for potential InternalError with some specific JVM options. Based on its Javadoc,
+            // MemoryPoolMXBean.getUsage() should return null, not throwing InternalError, so it seems to be a JVM bug.
+            return null;
+        }
+    }
+
+    private static void addBufferPoolMemoryInfo(Map<String, List<MemoryEntryVO>> memoryInfoMap) {
+        try {
+            List<MemoryEntryVO> bufferPoolMemEntries = new ArrayList<MemoryEntryVO>();
+            @SuppressWarnings("rawtypes")
+            Class bufferPoolMXBeanClass = Class.forName("java.lang.management.BufferPoolMXBean");
+            @SuppressWarnings("unchecked")
+            List<BufferPoolMXBean> bufferPoolMXBeans = ManagementFactory.getPlatformMXBeans(bufferPoolMXBeanClass);
+            for (BufferPoolMXBean mbean : bufferPoolMXBeans) {
+                long used = mbean.getMemoryUsed();
+                long total = mbean.getTotalCapacity();
+                bufferPoolMemEntries
+                        .add(new MemoryEntryVO(TYPE_BUFFER_POOL, mbean.getName(), used, total, Long.MIN_VALUE));
+            }
+            memoryInfoMap.put(TYPE_BUFFER_POOL, bufferPoolMemEntries);
+        } catch (ClassNotFoundException e) {
+            // ignore
+        }
+    }
+
+    private static MemoryEntryVO createMemoryEntryVO(String type, String name, MemoryUsage memoryUsage) {
+        return new MemoryEntryVO(type, name, memoryUsage.getUsed(), memoryUsage.getCommitted(), memoryUsage.getMax());
+    }
+}
\ No newline at end of file

core/src/main/java/com/taobao/arthas/core/command/monitor200/MonitorAdviceListener.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.core.advisor.Advice;
+import com.taobao.arthas.core.advisor.AdviceListenerAdapter;
+import com.taobao.arthas.core.advisor.ArthasMethod;
+import com.taobao.arthas.core.command.express.ExpressException;
+import com.taobao.arthas.core.command.model.MonitorModel;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.arthas.core.util.ThreadLocalWatch;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static com.taobao.arthas.core.util.ArthasCheckUtils.isEquals;
+
+/**
+ * 输出的内容格式为:<br/>
+ * <style type="text/css">
+ * table, th, td {
+ * borders:1px solid #cccccc;
+ * borders-collapse:collapse;
+ * }
+ * </style>
+ * <table>
+ * <tr>
+ * <th>时间戳</th>
+ * <th>统计周期(s)</th>
+ * <th>类全路径</th>
+ * <th>方法名</th>
+ * <th>调用总次数</th>
+ * <th>成功次数</th>
+ * <th>失败次数</th>
+ * <th>平均耗时(ms)</th>
+ * <th>失败率</th>
+ * </tr>
+ * <tr>
+ * <td>2012-11-07 05:00:01</td>
+ * <td>120</td>
+ * <td>com.taobao.item.ItemQueryServiceImpl</td>
+ * <td>queryItemForDetail</td>
+ * <td>1500</td>
+ * <td>1000</td>
+ * <td>500</td>
+ * <td>15</td>
+ * <td>30%</td>
+ * </tr>
+ * <tr>
+ * <td>2012-11-07 05:00:01</td>
+ * <td>120</td>
+ * <td>com.taobao.item.ItemQueryServiceImpl</td>
+ * <td>queryItemById</td>
+ * <td>900</td>
+ * <td>900</td>
+ * <td>0</td>
+ * <td>7</td>
+ * <td>0%</td>
+ * </tr>
+ * </table>
+ *
+ * @author beiwei30 on 28/11/2016.
+ */
+class MonitorAdviceListener extends AdviceListenerAdapter {
+    // 输出定时任务
+    private Timer timer;
+    private static final Logger logger = LoggerFactory.getLogger(MonitorAdviceListener.class);
+    // 监控数据
+    private ConcurrentHashMap<Key, AtomicReference<MonitorData>> monitorData = new ConcurrentHashMap<Key, AtomicReference<MonitorData>>();
+    private final ThreadLocalWatch threadLocalWatch = new ThreadLocalWatch();
+    private final ThreadLocal<Boolean> conditionResult = new ThreadLocal<Boolean>() {
+        @Override
+        protected Boolean initialValue() {
+            return true;
+        }
+    };
+    private MonitorCommand command;
+    private CommandProcess process;
+
+    MonitorAdviceListener(MonitorCommand command, CommandProcess process, boolean verbose) {
+        this.command = command;
+        this.process = process;
+        super.setVerbose(verbose);
+    }
+
+    @Override
+    public synchronized void create() {
+        if (timer == null) {
+            timer = new Timer("Timer-for-arthas-monitor-" + process.session().getSessionId(), true);
+            timer.scheduleAtFixedRate(new MonitorTimer(monitorData, process, command.getNumberOfLimit()),
+                    0, command.getCycle() * 1000L);
+        }
+    }
+
+    @Override
+    public synchronized void destroy() {
+        if (null != timer) {
+            timer.cancel();
+            timer = null;
+        }
+    }
+
+    @Override
+    public void before(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args)
+            throws Throwable {
+        threadLocalWatch.start();
+        if (!StringUtils.isEmpty(this.command.getConditionExpress()) && command.isBefore()) {
+            Advice advice = Advice.newForBefore(loader, clazz, method, target, args);
+            long cost = threadLocalWatch.cost();
+            this.conditionResult.set(isConditionMet(this.command.getConditionExpress(), advice, cost));
+            //重新计算执行方法的耗时(排除执行condition-express耗时)
+            threadLocalWatch.start();
+        }
+    }
+
+    @Override
+    public void afterReturning(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target,
+                               Object[] args, Object returnObject) throws Throwable {
+        finishing(clazz, method, false, Advice.newForAfterReturning(loader, clazz, method, target, args, returnObject));
+    }
+
+    @Override
+    public void afterThrowing(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target,
+                              Object[] args, Throwable throwable) {
+        finishing(clazz, method, true, Advice.newForAfterThrowing(loader, clazz, method, target, args, throwable));
+    }
+
+    private void finishing(Class<?> clazz, ArthasMethod method, boolean isThrowing, Advice advice) {
+        double cost = threadLocalWatch.costInMillis();
+
+        if (command.isBefore()) {
+            if (!this.conditionResult.get()) {
+                return;
+            }
+        } else {
+            try {
+                //不满足condition-express的不纳入统计
+                if (!isConditionMet(this.command.getConditionExpress(), advice, cost)) {
+                    return;
+                }
+            } catch (ExpressException e) {
+                //condition-express执行错误的不纳入统计
+                logger.warn("monitor execute condition-express failed.", e);
+                return;
+            }
+        }
+
+        final Key key = new Key(clazz.getName(), method.getName());
+
+        while (true) {
+            AtomicReference<MonitorData> value = monitorData.get(key);
+            if (null == value) {
+                monitorData.putIfAbsent(key, new AtomicReference<MonitorData>(new MonitorData()));
+                continue;
+            }
+
+            while (true) {
+                MonitorData oData = value.get();
+                MonitorData nData = new MonitorData();
+                nData.setCost(oData.getCost() + cost);
+                if (isThrowing) {
+                    nData.setFailed(oData.getFailed() + 1);
+                    nData.setSuccess(oData.getSuccess());
+                } else {
+                    nData.setFailed(oData.getFailed());
+                    nData.setSuccess(oData.getSuccess() + 1);
+                }
+                nData.setTotal(oData.getTotal() + 1);
+                if (value.compareAndSet(oData, nData)) {
+                    break;
+                }
+            }
+            break;
+        }
+    }
+
+    private class MonitorTimer extends TimerTask {
+        private Map<Key, AtomicReference<MonitorData>> monitorData;
+        private CommandProcess process;
+        private int limit;
+
+        MonitorTimer(Map<Key, AtomicReference<MonitorData>> monitorData, CommandProcess process, int limit) {
+            this.monitorData = monitorData;
+            this.process = process;
+            this.limit = limit;
+        }
+
+        @Override
+        public void run() {
+            if (monitorData.isEmpty()) {
+                return;
+            }
+            // 超过次数上限，则不再输出，命令终止
+            if (process.times().getAndIncrement() >= limit) {
+                this.cancel();
+                abortProcess(process, limit);
+                return;
+            }
+
+            List<MonitorData> monitorDataList = new ArrayList<MonitorData>(monitorData.size());
+            for (Map.Entry<Key, AtomicReference<MonitorData>> entry : monitorData.entrySet()) {
+                final AtomicReference<MonitorData> value = entry.getValue();
+
+                MonitorData data;
+                while (true) {
+                    data = value.get();
+                    //swap monitor data to new instance
+                    if (value.compareAndSet(data, new MonitorData())) {
+                        break;
+                    }
+                }
+
+                if (null != data) {
+                    data.setClassName(entry.getKey().getClassName());
+                    data.setMethodName(entry.getKey().getMethodName());
+                    monitorDataList.add(data);
+                }
+            }
+            process.appendResult(new MonitorModel(monitorDataList));
+        }
+
+    }
+
+    /**
+     * 数据监控用的Key
+     *
+     * @author vlinux
+     */
+    private static class Key {
+        private final String className;
+        private final String methodName;
+
+        Key(String className, String behaviorName) {
+            this.className = className;
+            this.methodName = behaviorName;
+        }
+
+        public String getClassName() {
+            return className;
+        }
+
+        public String getMethodName() {
+            return methodName;
+        }
+
+        @Override
+        public int hashCode() {
+            return className.hashCode() + methodName.hashCode();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (!(obj instanceof Key)) {
+                return false;
+            }
+            Key okey = (Key) obj;
+            return isEquals(okey.className, className) && isEquals(okey.methodName, methodName);
+        }
+
+    }
+
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/MonitorCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+
+import com.taobao.arthas.core.GlobalOptions;
+import com.taobao.arthas.core.advisor.AdviceListener;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.shell.handlers.Handler;
+import com.taobao.arthas.core.util.SearchUtils;
+import com.taobao.arthas.core.util.matcher.Matcher;
+import com.taobao.middleware.cli.annotations.Argument;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+/**
+ * 监控请求命令<br/>
+ * @author vlinux
+ */
+@Name("monitor")
+@Summary("Monitor method execution statistics, e.g. total/success/failure count, average rt, fail rate, etc. ")
+@Description("\nExamples:\n" +
+        "  monitor org.apache.commons.lang.StringUtils isBlank\n" +
+        "  monitor org.apache.commons.lang.StringUtils isBlank -c 5\n" +
+        "  monitor org.apache.commons.lang.StringUtils isBlank params[0]!=null\n" +
+        "  monitor -b org.apache.commons.lang.StringUtils isBlank params[0]!=null\n" +
+        "  monitor -E org\\.apache\\.commons\\.lang\\.StringUtils isBlank\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "monitor")
+public class MonitorCommand extends EnhancerCommand {
+
+    private String classPattern;
+    private String methodPattern;
+    private String conditionExpress;
+    private int cycle = 60;
+    private boolean isRegEx = false;
+    private int numberOfLimit = 100;
+    private boolean isBefore = false;
+
+    @Argument(argName = "class-pattern", index = 0)
+    @Description("Path and classname of Pattern Matching")
+    public void setClassPattern(String classPattern) {
+        this.classPattern = classPattern;
+    }
+
+    @Argument(argName = "method-pattern", index = 1)
+    @Description("Method of Pattern Matching")
+    public void setMethodPattern(String methodPattern) {
+        this.methodPattern = methodPattern;
+    }
+
+    @Argument(argName = "condition-express", index = 2, required = false)
+    @Description(Constants.CONDITION_EXPRESS)
+    public void setConditionExpress(String conditionExpress) {
+        this.conditionExpress = conditionExpress;
+    }
+
+    @Option(shortName = "c", longName = "cycle")
+    @Description("The monitor interval (in seconds), 60 seconds by default")
+    public void setCycle(int cycle) {
+        this.cycle = cycle;
+    }
+
+    @Option(shortName = "E", longName = "regex", flag = true)
+    @Description("Enable regular expression to match (wildcard matching by default)")
+    public void setRegEx(boolean regEx) {
+        isRegEx = regEx;
+    }
+
+    @Option(shortName = "n", longName = "limits")
+    @Description("Threshold of execution times")
+    public void setNumberOfLimit(int numberOfLimit) {
+        this.numberOfLimit = numberOfLimit;
+    }
+
+    @Option(shortName = "b", longName = "before", flag = true)
+    @Description("Evaluate the condition-express before method invoke")
+    public void setBefore(boolean before) {
+        isBefore = before;
+    }
+
+    public String getClassPattern() {
+        return classPattern;
+    }
+
+    public String getMethodPattern() {
+        return methodPattern;
+    }
+
+    public String getConditionExpress() {
+        return conditionExpress;
+    }
+
+    public int getCycle() {
+        return cycle;
+    }
+
+    public boolean isRegEx() {
+        return isRegEx;
+    }
+
+    public int getNumberOfLimit() {
+        return numberOfLimit;
+    }
+
+    public boolean isBefore() {
+        return isBefore;
+    }
+
+    @Override
+    protected Matcher getClassNameMatcher() {
+        if (classNameMatcher == null) {
+            classNameMatcher = SearchUtils.classNameMatcher(getClassPattern(), isRegEx());
+        }
+        return classNameMatcher;
+    }
+
+    @Override
+    protected Matcher getClassNameExcludeMatcher() {
+        if (classNameExcludeMatcher == null && getExcludeClassPattern() != null) {
+            classNameExcludeMatcher = SearchUtils.classNameMatcher(getExcludeClassPattern(), isRegEx());
+        }
+        return classNameExcludeMatcher;
+    }
+
+    @Override
+    protected Matcher getMethodNameMatcher() {
+        if (methodNameMatcher == null) {
+            methodNameMatcher = SearchUtils.classNameMatcher(getMethodPattern(), isRegEx());
+        }
+        return methodNameMatcher;
+    }
+
+    @Override
+    protected AdviceListener getAdviceListener(CommandProcess process) {
+        final AdviceListener listener = new MonitorAdviceListener(this, process, GlobalOptions.verbose || this.verbose);
+        /*
+         * 通过handle回调，在suspend时停止timer，resume时重启timer
+         */
+        process.suspendHandler(new Handler<Void>() {
+            @Override
+            public void handle(Void event) {
+                listener.destroy();
+            }
+        });
+        process.resumeHandler(new Handler<Void>() {
+            @Override
+            public void handle(Void event) {
+                listener.create();
+            }
+        });
+        return listener;
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/MonitorData.java

+package com.taobao.arthas.core.command.monitor200;
+
+/**
+ * 数据监控用的value for MonitorCommand
+ *
+ * @author vlinux
+ */
+public class MonitorData {
+    private String className;
+    private String methodName;
+    private int total;
+    private int success;
+    private int failed;
+    private double cost;
+
+    public String getClassName() {
+        return className;
+    }
+
+    public void setClassName(String className) {
+        this.className = className;
+    }
+
+    public String getMethodName() {
+        return methodName;
+    }
+
+    public void setMethodName(String methodName) {
+        this.methodName = methodName;
+    }
+
+    public int getTotal() {
+        return total;
+    }
+
+    public void setTotal(int total) {
+        this.total = total;
+    }
+
+    public int getSuccess() {
+        return success;
+    }
+
+    public void setSuccess(int success) {
+        this.success = success;
+    }
+
+    public int getFailed() {
+        return failed;
+    }
+
+    public void setFailed(int failed) {
+        this.failed = failed;
+    }
+
+    public double getCost() {
+        return cost;
+    }
+
+    public void setCost(double cost) {
+        this.cost = cost;
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/PathTraceAdviceListener.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.shell.command.CommandProcess;
+
+/**
+ * @author ralf0131 2017-01-05 13:59.
+ */
+public class PathTraceAdviceListener extends AbstractTraceAdviceListener {
+
+    public PathTraceAdviceListener(TraceCommand command, CommandProcess process) {
+        super(command, process);
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/PerfCounterCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.common.JavaVersionUtils;
+import com.taobao.arthas.common.PidUtils;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.model.PerfCounterModel;
+import com.taobao.arthas.core.command.model.PerfCounterVO;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import sun.management.counter.Counter;
+import sun.management.counter.perf.PerfInstrumentation;
+
+/**
+ * @see sun.misc.Perf
+ * @see sun.management.counter.perf.PerfInstrumentation
+ * @author hengyunabc 2020-02-16
+ */
+@Name("perfcounter")
+@Summary("Display the perf counter information.")
+@Description("\nExamples:\n" +
+        "  perfcounter\n" +
+        "  perfcounter -d\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "perfcounter")
+public class PerfCounterCommand extends AnnotatedCommand {
+    private static final Logger logger = LoggerFactory.getLogger(PerfCounterCommand.class);
+    private static Object perfObject;
+    private static Method attachMethod;
+
+    private boolean details;
+
+    @Option(shortName = "d", longName = "details", flag = true)
+    @Description("print all perf counter details")
+    public void setDetails(boolean details) {
+        this.details = details;
+    }
+
+    @Override
+    public void process(CommandProcess process) {
+        List<Counter> perfCounters = getPerfCounters();
+        if (perfCounters.isEmpty()) {
+            process.end(1,
+                    "please check arthas log. if java version >=9 , try to add jvm options when start your process: "
+                            + "--add-opens java.base/jdk.internal.perf=ALL-UNNAMED "
+                            + "--add-exports java.base/jdk.internal.perf=ALL-UNNAMED");
+            return;
+        }
+
+        List<PerfCounterVO> perfCounterVOs = new ArrayList<PerfCounterVO>();
+        for (Counter counter : perfCounters) {
+            PerfCounterVO perfCounterVO = new PerfCounterVO(counter.getName(), counter.getValue());
+            if (details) {
+                perfCounterVO.setUnits(counter.getUnits().toString());
+                perfCounterVO.setVariability(counter.getVariability().toString());
+            }
+            perfCounterVOs.add(perfCounterVO);
+        }
+        process.appendResult(new PerfCounterModel(perfCounterVOs, details));
+        process.end();
+    }
+
+    private static List<Counter> getPerfCounters() {
+
+        /**
+         * <pre>
+         * Perf p = Perf.getPerf();
+         * ByteBuffer buffer = p.attach(pid, "r");
+         * </pre>
+         */
+        try {
+            if (perfObject == null) {
+                // jdk8
+                String perfClassName = "sun.misc.Perf";
+                // jdk 11
+                if (!JavaVersionUtils.isLessThanJava9()) {
+                    perfClassName = "jdk.internal.perf.Perf";
+                }
+
+                Class<?> perfClass = ClassLoader.getSystemClassLoader().loadClass(perfClassName);
+                Method getPerfMethod = perfClass.getDeclaredMethod("getPerf");
+                perfObject = getPerfMethod.invoke(null);
+            }
+
+            if (attachMethod == null) {
+                attachMethod = perfObject.getClass().getDeclaredMethod("attach",
+                        new Class<?>[] { int.class, String.class });
+            }
+
+            ByteBuffer buffer = (ByteBuffer) attachMethod.invoke(perfObject,
+                    new Object[] { (int) PidUtils.currentLongPid(), "r" });
+
+            PerfInstrumentation perfInstrumentation = new PerfInstrumentation(buffer);
+            return perfInstrumentation.getAllCounters();
+        } catch (Throwable e) {
+            logger.error("get perf counter error", e);
+        }
+        return Collections.emptyList();
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/StackAdviceListener.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.core.advisor.Advice;
+import com.taobao.arthas.core.advisor.AdviceListenerAdapter;
+import com.taobao.arthas.core.advisor.ArthasMethod;
+import com.taobao.arthas.core.command.model.StackModel;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.LogUtil;
+import com.taobao.arthas.core.util.ThreadLocalWatch;
+import com.taobao.arthas.core.util.ThreadUtil;
+
+import java.util.Date;
+
+/**
+ * @author beiwei30 on 29/11/2016.
+ */
+public class StackAdviceListener extends AdviceListenerAdapter {
+    private static final Logger logger = LoggerFactory.getLogger(StackAdviceListener.class);
+
+    private final ThreadLocalWatch threadLocalWatch = new ThreadLocalWatch();
+    private StackCommand command;
+    private CommandProcess process;
+
+    public StackAdviceListener(StackCommand command, CommandProcess process, boolean verbose) {
+        this.command = command;
+        this.process = process;
+        super.setVerbose(verbose);
+    }
+
+    @Override
+    public void before(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args)
+            throws Throwable {
+        // 开始计算本次方法调用耗时
+        threadLocalWatch.start();
+    }
+
+    @Override
+    public void afterThrowing(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args,
+                              Throwable throwable) throws Throwable {
+        Advice advice = Advice.newForAfterThrowing(loader, clazz, method, target, args, throwable);
+        finishing(advice);
+    }
+
+    @Override
+    public void afterReturning(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args,
+                               Object returnObject) throws Throwable {
+        Advice advice = Advice.newForAfterReturning(loader, clazz, method, target, args, returnObject);
+        finishing(advice);
+    }
+
+    private void finishing(Advice advice) {
+        // 本次调用的耗时
+        try {
+            double cost = threadLocalWatch.costInMillis();
+            boolean conditionResult = isConditionMet(command.getConditionExpress(), advice, cost);
+            if (this.isVerbose()) {
+                process.write("Condition express: " + command.getConditionExpress() + " , result: " + conditionResult + "\n");
+            }
+            if (conditionResult) {
+                // TODO: concurrency issues for process.write
+                StackModel stackModel = ThreadUtil.getThreadStackModel(advice.getLoader(), Thread.currentThread());
+                stackModel.setTs(new Date());
+                process.appendResult(stackModel);
+                process.times().incrementAndGet();
+                if (isLimitExceeded(command.getNumberOfLimit(), process.times().get())) {
+                    abortProcess(process, command.getNumberOfLimit());
+                }
+            }
+        } catch (Throwable e) {
+            logger.warn("stack failed.", e);
+            process.end(-1, "stack failed, condition is: " + command.getConditionExpress() + ", " + e.getMessage()
+                          + ", visit " + LogUtil.loggingFile() + " for more details.");
+        }
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/StackCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.GlobalOptions;
+import com.taobao.arthas.core.advisor.AdviceListener;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.SearchUtils;
+import com.taobao.arthas.core.util.matcher.Matcher;
+import com.taobao.middleware.cli.annotations.Argument;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+/**
+ * Jstack命令<br/>
+ * 负责输出当前方法执行上下文
+ *
+ * @author vlinux
+ * @author hengyunabc 2016-10-31
+ */
+@Name("stack")
+@Summary("Display the stack trace for the specified class and method")
+@Description(Constants.EXPRESS_DESCRIPTION + Constants.EXAMPLE +
+        "  stack org.apache.commons.lang.StringUtils isBlank\n" +
+        "  stack *StringUtils isBlank\n" +
+        "  stack *StringUtils isBlank params[0].length==1\n" +
+        "  stack *StringUtils isBlank '#cost>100'\n" +
+        "  stack -E org\\.apache\\.commons\\.lang\\.StringUtils isBlank\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "stack")
+public class StackCommand extends EnhancerCommand {
+    private String classPattern;
+    private String methodPattern;
+    private String conditionExpress;
+    private boolean isRegEx = false;
+    private int numberOfLimit = 100;
+
+    @Argument(index = 0, argName = "class-pattern")
+    @Description("Path and classname of Pattern Matching")
+    public void setClassPattern(String classPattern) {
+        this.classPattern = classPattern;
+    }
+
+    @Argument(index = 1, argName = "method-pattern", required = false)
+    @Description("Method of Pattern Matching")
+    public void setMethodPattern(String methodPattern) {
+        this.methodPattern = methodPattern;
+    }
+
+    @Argument(index = 2, argName = "condition-express", required = false)
+    @Description(Constants.CONDITION_EXPRESS)
+    public void setConditionExpress(String conditionExpress) {
+        this.conditionExpress = conditionExpress;
+    }
+
+    @Option(shortName = "E", longName = "regex", flag = true)
+    @Description("Enable regular expression to match (wildcard matching by default)")
+    public void setRegEx(boolean regEx) {
+        isRegEx = regEx;
+    }
+
+    @Option(shortName = "n", longName = "limits")
+    @Description("Threshold of execution times")
+    public void setNumberOfLimit(int numberOfLimit) {
+        this.numberOfLimit = numberOfLimit;
+    }
+
+    public String getClassPattern() {
+        return classPattern;
+    }
+
+    public String getMethodPattern() {
+        return methodPattern;
+    }
+
+    public String getConditionExpress() {
+        return conditionExpress;
+    }
+
+    public boolean isRegEx() {
+        return isRegEx;
+    }
+
+    public int getNumberOfLimit() {
+        return numberOfLimit;
+    }
+
+    @Override
+    protected Matcher getClassNameMatcher() {
+        if (classNameMatcher == null) {
+            classNameMatcher = SearchUtils.classNameMatcher(getClassPattern(), isRegEx());
+        }
+        return classNameMatcher;
+    }
+
+    @Override
+    protected Matcher getClassNameExcludeMatcher() {
+        if (classNameExcludeMatcher == null && getExcludeClassPattern() != null) {
+            classNameExcludeMatcher = SearchUtils.classNameMatcher(getExcludeClassPattern(), isRegEx());
+        }
+        return classNameExcludeMatcher;
+    }
+
+    @Override
+    protected Matcher getMethodNameMatcher() {
+        if (methodNameMatcher == null) {
+            methodNameMatcher = SearchUtils.classNameMatcher(getMethodPattern(), isRegEx());
+        }
+        return methodNameMatcher;
+    }
+
+    @Override
+    protected AdviceListener getAdviceListener(CommandProcess process) {
+        return new StackAdviceListener(this, process, GlobalOptions.verbose || this.verbose);
+    }
+
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/ThreadCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.model.BlockingLockInfo;
+import com.taobao.arthas.core.command.model.BusyThreadInfo;
+import com.taobao.arthas.core.command.model.ThreadModel;
+import com.taobao.arthas.core.command.model.ThreadVO;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.shell.command.ExitStatus;
+import com.taobao.arthas.core.util.ArrayUtils;
+import com.taobao.arthas.core.util.CommandUtils;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.arthas.core.util.ThreadUtil;
+import com.taobao.middleware.cli.annotations.Argument;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import java.lang.Thread.State;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author hengyunabc 2015年12月7日 下午2:06:21
+ */
+@Name("thread")
+@Summary("Display thread info, thread stack")
+@Description(Constants.EXAMPLE +
+        "  thread\n" +
+        "  thread 51\n" +
+        "  thread -n -1\n" +
+        "  thread -n 5\n" +
+        "  thread -b\n" +
+        "  thread -i 2000\n" +
+        "  thread --state BLOCKED\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "thread")
+public class ThreadCommand extends AnnotatedCommand {
+    private static Set<String> states = null;
+    private static ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+
+    private long id = -1;
+    private Integer topNBusy = null;
+    private boolean findMostBlockingThread = false;
+    private int sampleInterval = 200;
+    private String state;
+
+    private boolean lockedMonitors = false;
+    private boolean lockedSynchronizers = false;
+    private boolean all = false;
+
+    static {
+        states = new HashSet<String>(State.values().length);
+        for (State state : State.values()) {
+            states.add(state.name());
+        }
+    }
+
+    @Argument(index = 0, required = false, argName = "id")
+    @Description("Show thread stack")
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    @Option(longName = "all", flag = true)
+    @Description("Display all thread results instead of the first page")
+    public void setAll(boolean all) {
+        this.all = all;
+    }
+
+    @Option(shortName = "n", longName = "top-n-threads")
+    @Description("The number of thread(s) to show, ordered by cpu utilization, -1 to show all.")
+    public void setTopNBusy(Integer topNBusy) {
+        this.topNBusy = topNBusy;
+    }
+
+    @Option(shortName = "b", longName = "include-blocking-thread", flag = true)
+    @Description("Find the thread who is holding a lock that blocks the most number of threads.")
+    public void setFindMostBlockingThread(boolean findMostBlockingThread) {
+        this.findMostBlockingThread = findMostBlockingThread;
+    }
+
+    @Option(shortName = "i", longName = "sample-interval")
+    @Description("Specify the sampling interval (in ms) when calculating cpu usage.")
+    public void setSampleInterval(int sampleInterval) {
+        this.sampleInterval = sampleInterval;
+    }
+
+    @Option(longName = "state")
+    @Description("Display the thead filter by the state. NEW, RUNNABLE, TIMED_WAITING, WAITING, BLOCKED, TERMINATED is optional.")
+    public void setState(String state) {
+        this.state = state;
+    }
+
+    @Option(longName = "lockedMonitors", flag = true)
+    @Description("Find the thread info with lockedMonitors flag, default value is false.")
+    public void setLockedMonitors(boolean lockedMonitors) {
+        this.lockedMonitors = lockedMonitors;
+    }
+
+    @Option(longName = "lockedSynchronizers", flag = true)
+    @Description("Find the thread info with lockedSynchronizers flag, default value is false.")
+    public void setLockedSynchronizers(boolean lockedSynchronizers) {
+        this.lockedSynchronizers = lockedSynchronizers;
+    }
+
+    @Override
+    public void process(CommandProcess process) {
+        ExitStatus exitStatus;
+        if (id > 0) {
+            exitStatus = processThread(process);
+        } else if (topNBusy != null) {
+            exitStatus = processTopBusyThreads(process);
+        } else if (findMostBlockingThread) {
+            exitStatus = processBlockingThread(process);
+        } else {
+            exitStatus = processAllThreads(process);
+        }
+        CommandUtils.end(process, exitStatus);
+    }
+
+    private ExitStatus processAllThreads(CommandProcess process) {
+        List<ThreadVO> threads = ThreadUtil.getThreads();
+
+        // 统计各种线程状态
+        Map<State, Integer> stateCountMap = new LinkedHashMap<State, Integer>();
+        for (State s : State.values()) {
+            stateCountMap.put(s, 0);
+        }
+
+        for (ThreadVO thread : threads) {
+            State threadState = thread.getState();
+            Integer count = stateCountMap.get(threadState);
+            stateCountMap.put(threadState, count + 1);
+        }
+
+        boolean includeInternalThreads = true;
+        Collection<ThreadVO> resultThreads = new ArrayList<ThreadVO>();
+        if (!StringUtils.isEmpty(this.state)) {
+            this.state = this.state.toUpperCase();
+            if (states.contains(this.state)) {
+                includeInternalThreads = false;
+                for (ThreadVO thread : threads) {
+                    if (thread.getState() != null && state.equals(thread.getState().name())) {
+                        resultThreads.add(thread);
+                    }
+                }
+            } else {
+                return ExitStatus.failure(1, "Illegal argument, state should be one of " + states);
+            }
+        } else {
+            resultThreads = threads;
+        }
+
+        //thread stats
+        ThreadSampler threadSampler = new ThreadSampler();
+        threadSampler.setIncludeInternalThreads(includeInternalThreads);
+        threadSampler.sample(resultThreads);
+        threadSampler.pause(sampleInterval);
+        List<ThreadVO> threadStats = threadSampler.sample(resultThreads);
+
+        process.appendResult(new ThreadModel(threadStats, stateCountMap, all));
+        return ExitStatus.success();
+    }
+
+    private ExitStatus processBlockingThread(CommandProcess process) {
+        BlockingLockInfo blockingLockInfo = ThreadUtil.findMostBlockingLock();
+        if (blockingLockInfo.getThreadInfo() == null) {
+            return ExitStatus.failure(1, "No most blocking thread found!");
+        }
+        process.appendResult(new ThreadModel(blockingLockInfo));
+        return ExitStatus.success();
+    }
+
+    private ExitStatus processTopBusyThreads(CommandProcess process) {
+        ThreadSampler threadSampler = new ThreadSampler();
+        threadSampler.sample(ThreadUtil.getThreads());
+        threadSampler.pause(sampleInterval);
+        List<ThreadVO> threadStats = threadSampler.sample(ThreadUtil.getThreads());
+
+        int limit = Math.min(threadStats.size(), topNBusy);
+
+        List<ThreadVO> topNThreads = null;
+        if (limit > 0) {
+            topNThreads = threadStats.subList(0, limit);
+        } else { // -1 for all threads
+            topNThreads = threadStats;
+        }
+
+        List<Long> tids = new ArrayList<Long>(topNThreads.size());
+        for (ThreadVO thread : topNThreads) {
+            if (thread.getId() > 0) {
+                tids.add(thread.getId());
+            }
+        }
+
+        ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(ArrayUtils.toPrimitive(tids.toArray(new Long[0])), lockedMonitors, lockedSynchronizers);
+        if (tids.size()> 0 && threadInfos == null) {
+            return ExitStatus.failure(1, "get top busy threads failed");
+        }
+
+        //threadInfo with cpuUsage
+        List<BusyThreadInfo> busyThreadInfos = new ArrayList<BusyThreadInfo>(topNThreads.size());
+        for (ThreadVO thread : topNThreads) {
+            ThreadInfo threadInfo = findThreadInfoById(threadInfos, thread.getId());
+            BusyThreadInfo busyThread = new BusyThreadInfo(thread, threadInfo);
+            busyThreadInfos.add(busyThread);
+        }
+        process.appendResult(new ThreadModel(busyThreadInfos));
+        return ExitStatus.success();
+    }
+
+    private ThreadInfo findThreadInfoById(ThreadInfo[] threadInfos, long id) {
+        for (int i = 0; i < threadInfos.length; i++) {
+            ThreadInfo threadInfo = threadInfos[i];
+            if ( threadInfo.getThreadId() == id) {
+                return threadInfo;
+            }
+        }
+        return null;
+    }
+
+    private ExitStatus processThread(CommandProcess process) {
+        ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(new long[]{id}, lockedMonitors, lockedSynchronizers);
+        if (threadInfos == null || threadInfos.length < 1 || threadInfos[0] == null) {
+            return ExitStatus.failure(1, "thread do not exist! id: " + id);
+        }
+
+        process.appendResult(new ThreadModel(threadInfos[0]));
+        return ExitStatus.success();
+    }
+}
\ No newline at end of file

core/src/main/java/com/taobao/arthas/core/command/monitor200/ThreadSampler.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.command.model.ThreadVO;
+import sun.management.HotspotThreadMBean;
+import sun.management.ManagementFactoryHelper;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Thread cpu sampler
+ *
+ * @author gongdewei 2020/4/23
+ */
+public class ThreadSampler {
+
+    private static ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+    private static HotspotThreadMBean hotspotThreadMBean;
+    private static boolean hotspotThreadMBeanEnable = true;
+
+    private Map<ThreadVO, Long> lastCpuTimes = new HashMap<ThreadVO, Long>();
+
+    private long lastSampleTimeNanos;
+    private boolean includeInternalThreads = true;
+
+
+    public List<ThreadVO> sample(Collection<ThreadVO> originThreads) {
+
+        List<ThreadVO> threads = new ArrayList<ThreadVO>(originThreads);
+
+        // Sample CPU
+        if (lastCpuTimes.isEmpty()) {
+            lastSampleTimeNanos = System.nanoTime();
+            for (ThreadVO thread : threads) {
+                if (thread.getId() > 0) {
+                    long cpu = threadMXBean.getThreadCpuTime(thread.getId());
+                    lastCpuTimes.put(thread, cpu);
+                    thread.setTime(cpu / 1000000);
+                }
+            }
+
+            // add internal threads
+            Map<String, Long> internalThreadCpuTimes = getInternalThreadCpuTimes();
+            if (internalThreadCpuTimes != null) {
+                for (Map.Entry<String, Long> entry : internalThreadCpuTimes.entrySet()) {
+                    String key = entry.getKey();
+                    ThreadVO thread = createThreadVO(key);
+                    thread.setTime(entry.getValue() / 1000000);
+                    threads.add(thread);
+                    lastCpuTimes.put(thread, entry.getValue());
+                }
+            }
+
+            //sort by time
+            Collections.sort(threads, new Comparator<ThreadVO>() {
+                @Override
+                public int compare(ThreadVO o1, ThreadVO o2) {
+                    long l1 = o1.getTime();
+                    long l2 = o2.getTime();
+                    if (l1 < l2) {
+                        return 1;
+                    } else if (l1 > l2) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            });
+            return threads;
+        }
+
+        // Resample
+        long newSampleTimeNanos = System.nanoTime();
+        Map<ThreadVO, Long> newCpuTimes = new HashMap<ThreadVO, Long>(threads.size());
+        for (ThreadVO thread : threads) {
+            if (thread.getId() > 0) {
+                long cpu = threadMXBean.getThreadCpuTime(thread.getId());
+                newCpuTimes.put(thread, cpu);
+            }
+        }
+        // internal threads
+        Map<String, Long> newInternalThreadCpuTimes = getInternalThreadCpuTimes();
+        if (newInternalThreadCpuTimes != null) {
+            for (Map.Entry<String, Long> entry : newInternalThreadCpuTimes.entrySet()) {
+                ThreadVO threadVO = createThreadVO(entry.getKey());
+                threads.add(threadVO);
+                newCpuTimes.put(threadVO, entry.getValue());
+            }
+        }
+
+        // Compute delta time
+        final Map<ThreadVO, Long> deltas = new HashMap<ThreadVO, Long>(threads.size());
+        for (ThreadVO thread : newCpuTimes.keySet()) {
+            Long t = lastCpuTimes.get(thread);
+            if (t == null) {
+                t = 0L;
+            }
+            long time1 = t;
+            long time2 = newCpuTimes.get(thread);
+            if (time1 == -1) {
+                time1 = time2;
+            } else if (time2 == -1) {
+                time2 = time1;
+            }
+            long delta = time2 - time1;
+            deltas.put(thread, delta);
+        }
+
+        long sampleIntervalNanos = newSampleTimeNanos - lastSampleTimeNanos;
+
+        // Compute cpu usage
+        final HashMap<ThreadVO, Double> cpuUsages = new HashMap<ThreadVO, Double>(threads.size());
+        for (ThreadVO thread : threads) {
+            double cpu = sampleIntervalNanos == 0 ? 0 : (Math.rint(deltas.get(thread) * 10000.0 / sampleIntervalNanos) / 100.0);
+            cpuUsages.put(thread, cpu);
+        }
+
+        // Sort by CPU time : should be a rendering hint...
+        Collections.sort(threads, new Comparator<ThreadVO>() {
+            public int compare(ThreadVO o1, ThreadVO o2) {
+                long l1 = deltas.get(o1);
+                long l2 = deltas.get(o2);
+                if (l1 < l2) {
+                    return 1;
+                } else if (l1 > l2) {
+                    return -1;
+                } else {
+                    return 0;
+                }
+            }
+        });
+
+        for (ThreadVO thread : threads) {
+            //nanos to mills
+            long timeMills = newCpuTimes.get(thread) / 1000000;
+            long deltaTime = deltas.get(thread) / 1000000;
+            double cpu = cpuUsages.get(thread);
+
+            thread.setCpu(cpu);
+            thread.setTime(timeMills);
+            thread.setDeltaTime(deltaTime);
+        }
+        lastCpuTimes = newCpuTimes;
+        lastSampleTimeNanos = newSampleTimeNanos;
+
+        return threads;
+    }
+
+    private Map<String, Long> getInternalThreadCpuTimes() {
+        if (hotspotThreadMBeanEnable && includeInternalThreads) {
+            try {
+                if (hotspotThreadMBean == null) {
+                    hotspotThreadMBean = ManagementFactoryHelper.getHotspotThreadMBean();
+                }
+                return hotspotThreadMBean.getInternalThreadCpuTimes();
+            } catch (Throwable e) {
+                //ignore ex
+                hotspotThreadMBeanEnable = false;
+            }
+        }
+        return null;
+    }
+
+    private ThreadVO createThreadVO(String name) {
+        ThreadVO threadVO = new ThreadVO();
+        threadVO.setId(-1);
+        threadVO.setName(name);
+        threadVO.setPriority(-1);
+        threadVO.setDaemon(true);
+        threadVO.setInterrupted(false);
+        return threadVO;
+    }
+
+    public void pause(long mills) {
+        try {
+            Thread.sleep(mills);
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+    public boolean isIncludeInternalThreads() {
+        return includeInternalThreads;
+    }
+
+    public void setIncludeInternalThreads(boolean includeInternalThreads) {
+        this.includeInternalThreads = includeInternalThreads;
+    }
+
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TimeFragment.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.advisor.Advice;
+
+import java.util.Date;
+
+/**
+ * 时间碎片
+ */
+class TimeFragment {
+
+    public TimeFragment(Advice advice, Date gmtCreate, double cost) {
+        this.advice = advice;
+        this.gmtCreate = gmtCreate;
+        this.cost = cost;
+    }
+
+    private final Advice advice;
+    private final Date gmtCreate;
+    private final double cost;
+
+    public Advice getAdvice() {
+        return advice;
+    }
+
+    public Date getGmtCreate() {
+        return gmtCreate;
+    }
+
+    public double getCost() {
+        return cost;
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TimeTunnelAdviceListener.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.core.advisor.Advice;
+import com.taobao.arthas.core.advisor.AdviceListenerAdapter;
+import com.taobao.arthas.core.advisor.ArthasMethod;
+import com.taobao.arthas.core.command.express.ExpressException;
+import com.taobao.arthas.core.command.model.TimeFragmentVO;
+import com.taobao.arthas.core.command.model.TimeTunnelModel;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.LogUtil;
+import com.taobao.arthas.core.util.ThreadLocalWatch;
+
+import java.util.Collections;
+import java.util.Date;
+
+/**
+ * @author beiwei30 on 30/11/2016.
+ * @author hengyunabc 2020-05-20
+ */
+public class TimeTunnelAdviceListener extends AdviceListenerAdapter {
+    private static final Logger logger = LoggerFactory.getLogger(TimeTunnelAdviceListener.class);
+    private final ThreadLocal<ObjectStack> argsRef = new ThreadLocal<ObjectStack>() {
+        @Override
+        protected ObjectStack initialValue() {
+            return new ObjectStack(512);
+        }
+    };
+
+    private TimeTunnelCommand command;
+    private CommandProcess process;
+
+    // 第一次启动标记
+    private volatile boolean isFirst = true;
+
+    // 方法执行时间戳
+    private final ThreadLocalWatch threadLocalWatch = new ThreadLocalWatch();
+
+    public TimeTunnelAdviceListener(TimeTunnelCommand command, CommandProcess process, boolean verbose) {
+        this.command = command;
+        this.process = process;
+        super.setVerbose(verbose);
+    }
+
+    @Override
+    public void before(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args)
+            throws Throwable {
+        argsRef.get().push(args);
+        threadLocalWatch.start();
+    }
+
+    @Override
+    public void afterReturning(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args,
+                               Object returnObject) throws Throwable {
+        //取出入参时的 args，因为在函数执行过程中 args可能被修改
+        args = (Object[]) argsRef.get().pop();
+        afterFinishing(Advice.newForAfterReturning(loader, clazz, method, target, args, returnObject));
+    }
+
+    @Override
+    public void afterThrowing(ClassLoader loader, Class<?> clazz, ArthasMethod method, Object target, Object[] args,
+                              Throwable throwable) {
+        //取出入参时的 args，因为在函数执行过程中 args可能被修改
+        args = (Object[]) argsRef.get().pop();
+        afterFinishing(Advice.newForAfterThrowing(loader, clazz, method, target, args, throwable));
+    }
+
+    private void afterFinishing(Advice advice) {
+        double cost = threadLocalWatch.costInMillis();
+        TimeFragment timeTunnel = new TimeFragment(advice, new Date(), cost);
+
+        boolean match = false;
+        try {
+            match = isConditionMet(command.getConditionExpress(), advice, cost);
+            if (this.isVerbose()) {
+                process.write("Condition express: " + command.getConditionExpress() + " , result: " + match + "\n");
+            }
+        } catch (ExpressException e) {
+            logger.warn("tt failed.", e);
+            process.end(-1, "tt failed, condition is: " + command.getConditionExpress() + ", " + e.getMessage()
+                          + ", visit " + LogUtil.loggingFile() + " for more details.");
+        }
+
+        if (!match) {
+            return;
+        }
+
+        int index = command.putTimeTunnel(timeTunnel);
+
+        TimeFragmentVO timeFragmentVO = TimeTunnelCommand.createTimeFragmentVO(index, timeTunnel, command.getExpand());
+        TimeTunnelModel timeTunnelModel = new TimeTunnelModel()
+                .setTimeFragmentList(Collections.singletonList(timeFragmentVO))
+                .setFirst(isFirst);
+        process.appendResult(timeTunnelModel);
+
+        if (isFirst) {
+            isFirst = false;
+        }
+
+        process.times().incrementAndGet();
+        if (isLimitExceeded(command.getNumberOfLimit(), process.times().get())) {
+            abortProcess(process, command.getNumberOfLimit());
+        }
+    }
+
+    /**
+     * 
+     * <pre>
+     * 一个特殊的stack，为了追求效率，避免扩容。
+     * 因为这个stack的push/pop 并不一定成对调用，比如可能push执行了，但是后面的流程被中断了，pop没有被执行。
+     * 如果不固定大小，一直增长的话，极端情况下可能应用有内存问题。
+     * 如果到达容量，pos会重置，循环存储数据。所以使用这个Stack如果在极端情况下统计的数据会不准确，只用于monitor/watch等命令的计时。
+     * 
+     * </pre>
+     * 
+     * @author hengyunabc 2020-05-20
+     *
+     */
+    static class ObjectStack {
+        private Object[] array;
+        private int pos = 0;
+        private int cap;
+
+        public ObjectStack(int maxSize) {
+            array = new Object[maxSize];
+            cap = array.length;
+        }
+
+        public int size() {
+            return pos;
+        }
+
+        public void push(Object value) {
+            if (pos < cap) {
+                array[pos++] = value;
+            } else {
+                // if array is full, reset pos
+                pos = 0;
+                array[pos++] = value;
+            }
+        }
+
+        public Object pop() {
+            if (pos > 0) {
+                pos--;
+                Object object = array[pos];
+                array[pos] = null;
+                return object;
+            } else {
+                pos = cap;
+                pos--;
+                Object object = array[pos];
+                array[pos] = null;
+                return object;
+            }
+        }
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TimeTunnelCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.core.GlobalOptions;
+import com.taobao.arthas.core.advisor.Advice;
+import com.taobao.arthas.core.advisor.AdviceListener;
+import com.taobao.arthas.core.advisor.ArthasMethod;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.express.ExpressException;
+import com.taobao.arthas.core.command.express.ExpressFactory;
+import com.taobao.arthas.core.command.model.MessageModel;
+import com.taobao.arthas.core.command.model.ObjectVO;
+import com.taobao.arthas.core.command.model.RowAffectModel;
+import com.taobao.arthas.core.command.model.TimeFragmentVO;
+import com.taobao.arthas.core.command.model.TimeTunnelModel;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.shell.handlers.command.CommandInterruptHandler;
+import com.taobao.arthas.core.shell.handlers.shell.QExitHandler;
+import com.taobao.arthas.core.util.LogUtil;
+import com.taobao.arthas.core.util.SearchUtils;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.arthas.core.util.affect.RowAffect;
+import com.taobao.arthas.core.util.matcher.Matcher;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+import com.taobao.middleware.cli.annotations.Argument;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.lang.Integer.toHexString;
+import static java.lang.String.format;
+
+/**
+ * 时光隧道命令<br/>
+ * 参数w/d依赖于参数i所传递的记录编号<br/>
+ *
+ * @author vlinux on 14/11/15.
+ */
+@Name("tt")
+@Summary("Time Tunnel")
+@Description(Constants.EXPRESS_DESCRIPTION + Constants.EXAMPLE +
+        "  tt -t *StringUtils isEmpty\n" +
+        "  tt -t *StringUtils isEmpty params[0].length==1\n" +
+        "  tt -l\n" +
+        "  tt -i 1000\n" +
+        "  tt -i 1000 -w params[0]\n" +
+        "  tt -i 1000 -p \n" +
+        "  tt -i 1000 -p --replay-times 3 --replay-interval 3000\n" +
+        "  tt -s '{params[0] > 1}' -w '{params}' \n" +
+        "  tt --delete-all\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "tt")
+public class TimeTunnelCommand extends EnhancerCommand {
+    // 时间隧道(时间碎片的集合)
+    // TODO 并非线程安全？
+    private static final Map<Integer, TimeFragment> timeFragmentMap = new LinkedHashMap<Integer, TimeFragment>();
+    // 时间碎片序列生成器
+    private static final AtomicInteger sequence = new AtomicInteger(1000);
+    // TimeTunnel the method call
+    private boolean isTimeTunnel = false;
+    private String classPattern;
+    private String methodPattern;
+    private String conditionExpress;
+    // list the TimeTunnel
+    private boolean isList = false;
+    private boolean isDeleteAll = false;
+    // index of TimeTunnel
+    private Integer index;
+    // expand of TimeTunnel
+    private Integer expand = 1;
+    // upper size limit
+    private Integer sizeLimit = 10 * 1024 * 1024;
+    // watch the index TimeTunnel
+    private String watchExpress = com.taobao.arthas.core.util.Constants.EMPTY_STRING;
+    private String searchExpress = com.taobao.arthas.core.util.Constants.EMPTY_STRING;
+    // play the index TimeTunnel
+    private boolean isPlay = false;
+    // delete the index TimeTunnel
+    private boolean isDelete = false;
+    private boolean isRegEx = false;
+    private int numberOfLimit = 100;
+    private int replayTimes = 1;
+    private long replayInterval = 1000L;
+    private static final Logger logger = LoggerFactory.getLogger(TimeTunnelCommand.class);
+
+    @Argument(index = 0, argName = "class-pattern", required = false)
+    @Description("Path and classname of Pattern Matching")
+    public void setClassPattern(String classPattern) {
+        this.classPattern = classPattern;
+    }
+
+    @Argument(index = 1, argName = "method-pattern", required = false)
+    @Description("Method of Pattern Matching")
+    public void setMethodPattern(String methodPattern) {
+        this.methodPattern = methodPattern;
+    }
+
+    @Argument(index = 2, argName = "condition-express", required = false)
+    @Description(Constants.CONDITION_EXPRESS)
+    public void setConditionExpress(String conditionExpress) {
+        this.conditionExpress = conditionExpress;
+    }
+
+    @Option(shortName = "t", longName = "time-tunnel", flag = true)
+    @Description("Record the method invocation within time fragments")
+    public void setTimeTunnel(boolean timeTunnel) {
+        isTimeTunnel = timeTunnel;
+    }
+
+    @Option(shortName = "l", longName = "list", flag = true)
+    @Description("List all the time fragments")
+    public void setList(boolean list) {
+        isList = list;
+    }
+
+    @Option(longName = "delete-all", flag = true)
+    @Description("Delete all the time fragments")
+    public void setDeleteAll(boolean deleteAll) {
+        isDeleteAll = deleteAll;
+    }
+
+    @Option(shortName = "i", longName = "index")
+    @Description("Display the detailed information from specified time fragment")
+    public void setIndex(Integer index) {
+        this.index = index;
+    }
+
+    @Option(shortName = "x", longName = "expand")
+    @Description("Expand level of object (1 by default)")
+    public void setExpand(Integer expand) {
+        this.expand = expand;
+    }
+
+    @Option(shortName = "M", longName = "sizeLimit")
+    @Description("Upper size limit in bytes for the result (10 * 1024 * 1024 by default)")
+    public void setSizeLimit(Integer sizeLimit) {
+        this.sizeLimit = sizeLimit;
+    }
+
+    @Option(shortName = "w", longName = "watch-express")
+    @Description(value = "watch the time fragment by ognl express.\n" + Constants.EXPRESS_EXAMPLES)
+    public void setWatchExpress(String watchExpress) {
+        this.watchExpress = watchExpress;
+    }
+
+    @Option(shortName = "s", longName = "search-express")
+    @Description("Search-expression, to search the time fragments by ognl express.\n" +
+            "The structure of 'advice' like conditional expression")
+    public void setSearchExpress(String searchExpress) {
+        this.searchExpress = searchExpress;
+    }
+
+    @Option(shortName = "p", longName = "play", flag = true)
+    @Description("Replay the time fragment specified by index")
+    public void setPlay(boolean play) {
+        isPlay = play;
+    }
+
+    @Option(shortName = "d", longName = "delete", flag = true)
+    @Description("Delete time fragment specified by index")
+    public void setDelete(boolean delete) {
+        isDelete = delete;
+    }
+
+    @Option(shortName = "E", longName = "regex", flag = true)
+    @Description("Enable regular expression to match (wildcard matching by default)")
+    public void setRegEx(boolean regEx) {
+        isRegEx = regEx;
+    }
+
+    @Option(shortName = "n", longName = "limits")
+    @Description("Threshold of execution times")
+    public void setNumberOfLimit(int numberOfLimit) {
+        this.numberOfLimit = numberOfLimit;
+    }
+
+
+    @Option(longName = "replay-times")
+    @Description("execution times when play tt")
+    public void setReplayTimes(int replayTimes) {
+        this.replayTimes = replayTimes;
+    }
+
+    @Option(longName = "replay-interval")
+    @Description("replay interval  for  play tt with option r greater than 1")
+    public void setReplayInterval(int replayInterval) {
+        this.replayInterval = replayInterval;
+    }
+
+
+    public boolean isRegEx() {
+        return isRegEx;
+    }
+
+    public String getMethodPattern() {
+        return methodPattern;
+    }
+
+    public String getClassPattern() {
+        return classPattern;
+    }
+
+    public String getConditionExpress() {
+        return conditionExpress;
+    }
+
+    public int getNumberOfLimit() {
+        return numberOfLimit;
+    }
+
+    public int getReplayTimes() {
+        return replayTimes;
+    }
+
+    public long getReplayInterval() {
+        return replayInterval;
+    }
+
+    public Integer getExpand() {
+        return expand;
+    }
+
+    private boolean hasWatchExpress() {
+        return !StringUtils.isEmpty(watchExpress);
+    }
+
+    private boolean hasSearchExpress() {
+        return !StringUtils.isEmpty(searchExpress);
+    }
+
+    /**
+     * 检查参数是否合法
+     */
+    private void checkArguments() {
+        // 检查d/p参数是否有i参数配套
+        if ((isDelete || isPlay) && null == index) {
+            throw new IllegalArgumentException("Time fragment index is expected, please type -i to specify");
+        }
+
+        // 在t参数下class-pattern,method-pattern
+        if (isTimeTunnel) {
+            if (StringUtils.isEmpty(classPattern)) {
+                throw new IllegalArgumentException("Class-pattern is expected, please type the wildcard expression to match");
+            }
+            if (StringUtils.isEmpty(methodPattern)) {
+                throw new IllegalArgumentException("Method-pattern is expected, please type the wildcard expression to match");
+            }
+        }
+
+        // 一个参数都没有是不行滴
+        if (null == index && !isTimeTunnel && !isDeleteAll && StringUtils.isEmpty(watchExpress)
+                && !isList && StringUtils.isEmpty(searchExpress)) {
+            throw new IllegalArgumentException("Argument(s) is/are expected, type 'help tt' to read usage");
+        }
+    }
+
+    /*
+     * 记录时间片段
+     */
+    int putTimeTunnel(TimeFragment tt) {
+        int indexOfSeq = sequence.getAndIncrement();
+        timeFragmentMap.put(indexOfSeq, tt);
+        return indexOfSeq;
+    }
+
+    @Override
+    public void process(final CommandProcess process) {
+        // 检查参数
+        checkArguments();
+
+        // ctrl-C support
+        process.interruptHandler(new CommandInterruptHandler(process));
+        // q exit support
+        process.stdinHandler(new QExitHandler(process));
+
+        if (isTimeTunnel) {
+            enhance(process);
+        } else if (isPlay) {
+            processPlay(process);
+        } else if (isList) {
+            processList(process);
+        } else if (isDeleteAll) {
+            processDeleteAll(process);
+        } else if (isDelete) {
+            processDelete(process);
+        } else if (hasSearchExpress()) {
+            processSearch(process);
+        } else if (index != null) {
+            if (hasWatchExpress()) {
+                processWatch(process);
+            } else {
+                processShow(process);
+            }
+        }
+    }
+
+    @Override
+    protected Matcher getClassNameMatcher() {
+        if (classNameMatcher == null) {
+            classNameMatcher = SearchUtils.classNameMatcher(getClassPattern(), isRegEx());
+        }
+        return classNameMatcher;
+    }
+
+    @Override
+    protected Matcher getClassNameExcludeMatcher() {
+        if (classNameExcludeMatcher == null && getExcludeClassPattern() != null) {
+            classNameExcludeMatcher = SearchUtils.classNameMatcher(getExcludeClassPattern(), isRegEx());
+        }
+        return classNameExcludeMatcher;
+    }
+
+    @Override
+    protected Matcher getMethodNameMatcher() {
+        if (methodNameMatcher == null) {
+            methodNameMatcher = SearchUtils.classNameMatcher(getMethodPattern(), isRegEx());
+        }
+        return methodNameMatcher;
+    }
+
+    @Override
+    protected AdviceListener getAdviceListener(CommandProcess process) {
+        return new TimeTunnelAdviceListener(this, process, GlobalOptions.verbose || this.verbose);
+    }
+
+    // 展示指定记录
+    private void processShow(CommandProcess process) {
+        RowAffect affect = new RowAffect();
+        try {
+            TimeFragment tf = timeFragmentMap.get(index);
+            if (null == tf) {
+                process.end(1, format("Time fragment[%d] does not exist.", index));
+                return;
+            }
+
+            TimeFragmentVO timeFragmentVO = createTimeFragmentVO(index, tf, expand);
+            TimeTunnelModel timeTunnelModel = new TimeTunnelModel()
+                    .setTimeFragment(timeFragmentVO)
+                    .setExpand(expand)
+                    .setSizeLimit(sizeLimit);
+            process.appendResult(timeTunnelModel);
+            affect.rCnt(1);
+            process.appendResult(new RowAffectModel(affect));
+            process.end();
+        } catch (Throwable e) {
+            logger.warn("tt failed.", e);
+            process.end(1, e.getMessage() + ", visit " + LogUtil.loggingFile() + " for more detail");
+        }
+    }
+
+    // 查看记录信息
+    private void processWatch(CommandProcess process) {
+        RowAffect affect = new RowAffect();
+        try {
+            final TimeFragment tf = timeFragmentMap.get(index);
+            if (null == tf) {
+                process.end(1, format("Time fragment[%d] does not exist.", index));
+                return;
+            }
+
+            Advice advice = tf.getAdvice();
+
+			Object value = ExpressFactory.unpooledExpress(advice.getLoader()).bind(advice).get(watchExpress);
+            TimeTunnelModel timeTunnelModel = new TimeTunnelModel()
+                    .setWatchValue(new ObjectVO(value, expand))
+                    .setExpand(expand)
+                    .setSizeLimit(sizeLimit);
+            process.appendResult(timeTunnelModel);
+
+            affect.rCnt(1);
+            process.appendResult(new RowAffectModel(affect));
+            process.end();
+        } catch (ExpressException e) {
+            logger.warn("tt failed.", e);
+            process.end(1, e.getMessage() + ", visit " + LogUtil.loggingFile() + " for more detail");
+        }
+    }
+
+    // do search timeFragmentMap
+    private void processSearch(CommandProcess process) {
+        RowAffect affect = new RowAffect();
+        try {
+            // 匹配的时间片段
+            Map<Integer, TimeFragment> matchingTimeSegmentMap = new LinkedHashMap<Integer, TimeFragment>();
+            for (Map.Entry<Integer, TimeFragment> entry : timeFragmentMap.entrySet()) {
+                int index = entry.getKey();
+                TimeFragment tf = entry.getValue();
+                Advice advice = tf.getAdvice();
+
+                // 搜索出匹配的时间片段
+                if ((ExpressFactory.threadLocalExpress(advice)).is(searchExpress)) {
+                    matchingTimeSegmentMap.put(index, tf);
+                }
+            }
+
+            if (hasWatchExpress()) {
+                // 执行watchExpress
+                Map<Integer, ObjectVO> searchResults = new LinkedHashMap<Integer, ObjectVO>();
+                for (Map.Entry<Integer, TimeFragment> entry : matchingTimeSegmentMap.entrySet()) {
+                    Object value = ExpressFactory.threadLocalExpress(entry.getValue().getAdvice()).get(watchExpress);
+                    searchResults.put(entry.getKey(), new ObjectVO(value, expand));
+                }
+
+                TimeTunnelModel timeTunnelModel = new TimeTunnelModel()
+                        .setWatchResults(searchResults)
+                        .setExpand(expand)
+                        .setSizeLimit(sizeLimit);
+                process.appendResult(timeTunnelModel);
+            } else {
+                // 单纯的列表格
+                List<TimeFragmentVO> timeFragmentList = createTimeTunnelVOList(matchingTimeSegmentMap);
+                process.appendResult(new TimeTunnelModel().setTimeFragmentList(timeFragmentList).setFirst(true));
+            }
+
+            affect.rCnt(matchingTimeSegmentMap.size());
+            process.appendResult(new RowAffectModel(affect));
+            process.end();
+        } catch (ExpressException e) {
+            logger.warn("tt failed.", e);
+            process.end(1, e.getMessage() + ", visit " + LogUtil.loggingFile() + " for more detail");
+        }
+    }
+
+    // 删除指定记录
+    private void processDelete(CommandProcess process) {
+        RowAffect affect = new RowAffect();
+        if (timeFragmentMap.remove(index) != null) {
+            affect.rCnt(1);
+        }
+        process.appendResult(new MessageModel(format("Time fragment[%d] successfully deleted.", index)));
+        process.appendResult(new RowAffectModel(affect));
+        process.end();
+    }
+
+    private void processDeleteAll(CommandProcess process) {
+        int count = timeFragmentMap.size();
+        RowAffect affect = new RowAffect(count);
+        timeFragmentMap.clear();
+        process.appendResult(new MessageModel("Time fragments are cleaned."));
+        process.appendResult(new RowAffectModel(affect));
+        process.end();
+    }
+
+    private void processList(CommandProcess process) {
+        RowAffect affect = new RowAffect();
+        List<TimeFragmentVO> timeFragmentList = createTimeTunnelVOList(timeFragmentMap);
+        process.appendResult(new TimeTunnelModel().setTimeFragmentList(timeFragmentList).setFirst(true));
+        affect.rCnt(timeFragmentMap.size());
+        process.appendResult(new RowAffectModel(affect));
+        process.end();
+    }
+
+    private List<TimeFragmentVO> createTimeTunnelVOList(Map<Integer, TimeFragment> timeFragmentMap) {
+        List<TimeFragmentVO> timeFragmentList = new ArrayList<TimeFragmentVO>(timeFragmentMap.size());
+        for (Map.Entry<Integer, TimeFragment> entry : timeFragmentMap.entrySet()) {
+            timeFragmentList.add(createTimeFragmentVO(entry.getKey(), entry.getValue(), expand));
+        }
+        return timeFragmentList;
+    }
+
+    public static TimeFragmentVO createTimeFragmentVO(Integer index, TimeFragment tf, Integer expand) {
+        Advice advice = tf.getAdvice();
+        String object = advice.getTarget() == null
+                ? "NULL"
+                : "0x" + toHexString(advice.getTarget().hashCode());
+
+        return new TimeFragmentVO()
+                .setIndex(index)
+                .setTimestamp(tf.getGmtCreate())
+                .setCost(tf.getCost())
+                .setParams(ObjectVO.array(advice.getParams(), expand))
+                .setReturn(advice.isAfterReturning())
+                .setReturnObj(new ObjectVO(advice.getReturnObj(), expand))
+                .setThrow(advice.isAfterThrowing())
+                .setThrowExp(new ObjectVO(advice.getThrowExp(), expand))
+                .setObject(object)
+                .setClassName(advice.getClazz().getName())
+                .setMethodName(advice.getMethod().getName());
+    }
+
+    /**
+     * 重放指定记录
+     */
+    private void processPlay(CommandProcess process) {
+        TimeFragment tf = timeFragmentMap.get(index);
+        if (null == tf) {
+            process.end(1, format("Time fragment[%d] does not exist.", index));
+            return;
+        }
+        Advice advice = tf.getAdvice();
+        ArthasMethod method = advice.getMethod();
+        boolean accessible = advice.getMethod().isAccessible();
+        try {
+            if (!accessible) {
+                method.setAccessible(true);
+            }
+            for (int i = 0; i < getReplayTimes(); i++) {
+                if (i > 0) {
+                    //wait for the next execution
+                    Thread.sleep(getReplayInterval());
+                    if (!process.isRunning()) {
+                        return;
+                    }
+                }
+                long beginTime = System.nanoTime();
+
+                //copy from tt record
+                TimeFragmentVO replayResult = createTimeFragmentVO(index, tf, expand);
+                replayResult.setTimestamp(new Date())
+                        .setCost(0)
+                        .setReturn(false)
+                        .setReturnObj(null)
+                        .setThrow(false)
+                        .setThrowExp(null);
+
+                try {
+                    //execute successful
+                    Object returnObj = method.invoke(advice.getTarget(), advice.getParams());
+                    double cost = (System.nanoTime() - beginTime) / 1000000.0;
+                    replayResult.setCost(cost)
+                            .setReturn(true)
+                            .setReturnObj(new ObjectVO(returnObj, expand));
+                } catch (Throwable t) {
+                    //throw exp
+                    double cost = (System.nanoTime() - beginTime) / 1000000.0;
+                    replayResult.setCost(cost)
+                            .setThrow(true)
+                            .setThrowExp(new ObjectVO(t, expand));
+                }
+
+                TimeTunnelModel timeTunnelModel = new TimeTunnelModel()
+                        .setReplayResult(replayResult)
+                        .setReplayNo(i + 1)
+                        .setExpand(expand)
+                        .setSizeLimit(sizeLimit);
+                process.appendResult(timeTunnelModel);
+            }
+            process.end();
+        } catch (Throwable t) {
+            logger.warn("tt replay failed.", t);
+            process.end(-1, "tt replay failed");
+        } finally {
+            method.setAccessible(accessible);
+        }
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TimeTunnelTable.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.command.model.ObjectVO;
+import com.taobao.arthas.core.command.model.TimeFragmentVO;
+import com.taobao.arthas.core.util.StringUtils;
+import com.taobao.arthas.core.view.ObjectView;
+import com.taobao.text.Decoration;
+import com.taobao.text.ui.Element;
+import com.taobao.text.ui.LabelElement;
+import com.taobao.text.ui.TableElement;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.reflect.InvocationTargetException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import static com.taobao.text.ui.Element.label;
+
+/**
+ * @author beiwei30 on 30/11/2016.
+ */
+public class TimeTunnelTable {
+    // 各列宽度
+    private static final int[] TABLE_COL_WIDTH = new int[]{
+            8, // index
+            20, // timestamp
+            10, // cost(ms)
+            8, // isRet
+            8, // isExp
+            15, // object address
+            30, // class
+            30, // method
+    };
+
+    // 各列名称
+    private static final String[] TABLE_COL_TITLE = new String[]{
+            "INDEX",
+            "TIMESTAMP",
+            "COST(ms)",
+            "IS-RET",
+            "IS-EXP",
+            "OBJECT",
+            "CLASS",
+            "METHOD"
+
+    };
+
+    static TableElement createTable() {
+        return new TableElement(TABLE_COL_WIDTH).leftCellPadding(1).rightCellPadding(1);
+    }
+
+    public static TableElement createDefaultTable() {
+        return new TableElement().leftCellPadding(1).rightCellPadding(1);
+    }
+
+    static TableElement fillTableHeader(TableElement table) {
+        LabelElement[] headers = new LabelElement[TABLE_COL_TITLE.length];
+        for (int i = 0; i < TABLE_COL_TITLE.length; ++i) {
+            headers[i] = label(TABLE_COL_TITLE[i]).style(Decoration.bold.bold());
+        }
+        table.row(true, headers);
+        return table;
+    }
+
+    // 绘制TimeTunnel表格
+    public static Element drawTimeTunnelTable(List<TimeFragmentVO> timeFragmentList, boolean withHeader){
+        TableElement table = createTable();
+        if (withHeader) {
+            fillTableHeader(table);
+        }
+        for (TimeFragmentVO tf : timeFragmentList) {
+            fillTableRow(table, tf);
+        }
+        return table;
+    }
+
+    // 填充表格行
+    static TableElement fillTableRow(TableElement table, TimeFragmentVO tf) {
+        return table.row(
+                "" + tf.getIndex(),
+                new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tf.getTimestamp()),
+                "" + tf.getCost(),
+                "" + tf.isReturn(),
+                "" + tf.isThrow(),
+                tf.getObject(),
+                StringUtils.substringAfterLast("." + tf.getClassName(), "."),
+                tf.getMethodName()
+        );
+    }
+
+    public static void drawTimeTunnel(TableElement table, TimeFragmentVO tf) {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        table.row("INDEX", "" + tf.getIndex())
+                .row("GMT-CREATE", sdf.format(tf.getTimestamp()))
+                .row("COST(ms)", "" + tf.getCost())
+                .row("OBJECT", tf.getObject())
+                .row("CLASS", tf.getClassName())
+                .row("METHOD", tf.getMethodName())
+                .row("IS-RETURN", "" + tf.isReturn())
+                .row("IS-EXCEPTION", "" + tf.isThrow());
+    }
+
+    public static void drawThrowException(TableElement table, TimeFragmentVO tf) {
+        if (tf.isThrow()) {
+            //noinspection ThrowableResultOfMethodCallIgnored
+            ObjectVO throwableVO = tf.getThrowExp();
+            if (throwableVO.needExpand()) {
+                table.row("THROW-EXCEPTION", new ObjectView(throwableVO).draw());
+            } else {
+                StringWriter stringWriter = new StringWriter();
+                PrintWriter printWriter = new PrintWriter(stringWriter);
+                try {
+                    ((Throwable) throwableVO.getObject()).printStackTrace(printWriter);
+                    table.row("THROW-EXCEPTION", stringWriter.toString());
+                } finally {
+                    printWriter.close();
+                }
+            }
+        }
+    }
+
+    public static void drawReturnObj(TableElement table, TimeFragmentVO tf, Integer sizeLimit) {
+        if (tf.isReturn()) {
+            if (tf.getReturnObj().needExpand()) {
+                table.row("RETURN-OBJ", new ObjectView(sizeLimit, tf.getReturnObj()).draw());
+            } else {
+                table.row("RETURN-OBJ", "" + StringUtils.objectToString(tf.getReturnObj()));
+            }
+        }
+    }
+
+    public static void drawParameters(TableElement table, ObjectVO[] params) {
+        if (params != null) {
+            int paramIndex = 0;
+            for (ObjectVO param : params) {
+                if (param.needExpand()) {
+                    table.row("PARAMETERS[" + paramIndex++ + "]", new ObjectView(param).draw());
+                } else {
+                    table.row("PARAMETERS[" + paramIndex++ + "]", "" + StringUtils.objectToString(param));
+                }
+            }
+        }
+    }
+
+    public static void drawWatchTableHeader(TableElement table) {
+        table.row(true, label("INDEX").style(Decoration.bold.bold()), label("SEARCH-RESULT")
+                .style(Decoration.bold.bold()));
+    }
+
+    public static void drawWatchResults(TableElement table, Map<Integer, ObjectVO> watchResults, Integer sizeLimit) {
+        for (Map.Entry<Integer, ObjectVO> entry : watchResults.entrySet()) {
+            ObjectVO objectVO = entry.getValue();
+            table.row("" + entry.getKey(), "" +
+                    (objectVO.needExpand() ? new ObjectView(sizeLimit, objectVO).draw() : StringUtils.objectToString(objectVO.getObject())));
+        }
+    }
+
+    public static TableElement drawPlayHeader(String className, String methodName, String objectAddress, int index,
+                                       TableElement table) {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        return table.row("RE-INDEX", "" + index)
+                .row("GMT-REPLAY", sdf.format(new Date()))
+                .row("OBJECT", objectAddress)
+                .row("CLASS", className)
+                .row("METHOD", methodName);
+    }
+
+    public static void drawPlayResult(TableElement table, ObjectVO returnObjVO,
+                               int sizeLimit, double cost) {
+        // 执行成功:输出成功状态
+        table.row("IS-RETURN", "" + true);
+        table.row("IS-EXCEPTION", "" + false);
+        table.row("COST(ms)", "" + cost);
+
+        // 执行成功:输出成功结果
+        if (returnObjVO.needExpand()) {
+            table.row("RETURN-OBJ", new ObjectView(sizeLimit, returnObjVO).draw());
+        } else {
+            table.row("RETURN-OBJ", "" + StringUtils.objectToString(returnObjVO.getObject()));
+        }
+    }
+
+    public static void drawPlayException(TableElement table, ObjectVO throwableVO) {
+        // 执行失败:输出失败状态
+        table.row("IS-RETURN", "" + false);
+        table.row("IS-EXCEPTION", "" + true);
+
+        // 执行失败:输出失败异常信息
+        Throwable cause;
+        Throwable t = (Throwable) throwableVO.getObject();
+        if (t instanceof InvocationTargetException) {
+            cause = t.getCause();
+        } else {
+            cause = t;
+        }
+
+        if (throwableVO.needExpand()) {
+            table.row("THROW-EXCEPTION", new ObjectView(cause, throwableVO.expandOrDefault()).draw());
+        } else {
+            StringWriter stringWriter = new StringWriter();
+            PrintWriter printWriter = new PrintWriter(stringWriter);
+            try {
+                cause.printStackTrace(printWriter);
+                table.row("THROW-EXCEPTION", stringWriter.toString());
+            } finally {
+                printWriter.close();
+            }
+        }
+    }
+
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TraceAdviceListener.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.advisor.InvokeTraceable;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+
+/**
+ * @author beiwei30 on 29/11/2016.
+ */
+public class TraceAdviceListener extends AbstractTraceAdviceListener implements InvokeTraceable {
+
+    /**
+     * Constructor
+     */
+    public TraceAdviceListener(TraceCommand command, CommandProcess process, boolean verbose) {
+        super(command, process);
+        super.setVerbose(verbose);
+    }
+
+    /**
+     * trace 会在被观测的方法体中，在每个方法调用前后插入字节码，所以方法调用开始，结束，抛异常的时候，都会回调下面的接口
+     */
+    @Override
+    public void invokeBeforeTracing(ClassLoader classLoader, String tracingClassName, String tracingMethodName, String tracingMethodDesc, int tracingLineNumber)
+            throws Throwable {
+        // normalize className later
+        threadLocalTraceEntity(classLoader).tree.begin(tracingClassName, tracingMethodName, tracingLineNumber, true);
+    }
+
+    @Override
+    public void invokeAfterTracing(ClassLoader classLoader, String tracingClassName, String tracingMethodName, String tracingMethodDesc, int tracingLineNumber)
+            throws Throwable {
+        threadLocalTraceEntity(classLoader).tree.end();
+    }
+
+    @Override
+    public void invokeThrowTracing(ClassLoader classLoader, String tracingClassName, String tracingMethodName, String tracingMethodDesc, int tracingLineNumber)
+            throws Throwable {
+        threadLocalTraceEntity(classLoader).tree.end(true);
+    }
+
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TraceCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.GlobalOptions;
+import com.taobao.arthas.core.advisor.AdviceListener;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.SearchUtils;
+import com.taobao.arthas.core.util.matcher.GroupMatcher;
+import com.taobao.arthas.core.util.matcher.Matcher;
+import com.taobao.arthas.core.util.matcher.RegexMatcher;
+import com.taobao.arthas.core.util.matcher.TrueMatcher;
+import com.taobao.arthas.core.util.matcher.WildcardMatcher;
+import com.taobao.middleware.cli.annotations.Argument;
+import com.taobao.middleware.cli.annotations.DefaultValue;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 调用跟踪命令<br/>
+ * 负责输出一个类中的所有方法调用路径
+ *
+ * @author vlinux on 15/5/27.
+ */
+// @formatter:off
+@Name("trace")
+@Summary("Trace the execution time of specified method invocation.")
+@Description(value = Constants.EXPRESS_DESCRIPTION + Constants.EXAMPLE +
+        "  trace org.apache.commons.lang.StringUtils isBlank\n" +
+        "  trace *StringUtils isBlank\n" +
+        "  trace *StringUtils isBlank params[0].length==1\n" +
+        "  trace *StringUtils isBlank '#cost>100'\n" +
+        "  trace -E org\\\\.apache\\\\.commons\\\\.lang\\\\.StringUtils isBlank\n" +
+        "  trace -E com.test.ClassA|org.test.ClassB method1|method2|method3\n" +
+        "  trace demo.MathGame run -n 5\n" +
+        "  trace demo.MathGame run --skipJDKMethod false\n" +
+        "  trace javax.servlet.Filter * --exclude-class-pattern com.demo.TestFilter\n" +
+        "  trace OuterClass$InnerClass *\n" +
+        Constants.WIKI + Constants.WIKI_HOME + "trace")
+//@formatter:on
+public class TraceCommand extends EnhancerCommand {
+
+    private String classPattern;
+    private String methodPattern;
+    private String conditionExpress;
+    private boolean isRegEx = false;
+    private int numberOfLimit = 100;
+    private List<String> pathPatterns;
+    private boolean skipJDKTrace;
+
+    @Argument(argName = "class-pattern", index = 0)
+    @Description("Class name pattern, use either '.' or '/' as separator")
+    public void setClassPattern(String classPattern) {
+        this.classPattern = classPattern;
+    }
+
+    @Argument(argName = "method-pattern", index = 1)
+    @Description("Method name pattern")
+    public void setMethodPattern(String methodPattern) {
+        this.methodPattern = methodPattern;
+    }
+
+    @Argument(argName = "condition-express", index = 2, required = false)
+    @Description(Constants.CONDITION_EXPRESS)
+    public void setConditionExpress(String conditionExpress) {
+        this.conditionExpress = conditionExpress;
+    }
+
+    @Option(shortName = "E", longName = "regex", flag = true)
+    @Description("Enable regular expression to match (wildcard matching by default)")
+    public void setRegEx(boolean regEx) {
+        isRegEx = regEx;
+    }
+
+    @Option(shortName = "n", longName = "limits")
+    @Description("Threshold of execution times")
+    public void setNumberOfLimit(int numberOfLimit) {
+        this.numberOfLimit = numberOfLimit;
+    }
+
+    @Option(shortName = "p", longName = "path", acceptMultipleValues = true)
+    @Description("path tracing pattern")
+    public void setPathPatterns(List<String> pathPatterns) {
+        this.pathPatterns = pathPatterns;
+    }
+
+    @Option(longName = "skipJDKMethod")
+    @DefaultValue("true")
+    @Description("skip jdk method trace, default value true.")
+    public void setSkipJDKTrace(boolean skipJDKTrace) {
+        this.skipJDKTrace = skipJDKTrace;
+    }
+
+    public String getClassPattern() {
+        return classPattern;
+    }
+
+    public String getMethodPattern() {
+        return methodPattern;
+    }
+
+    public String getConditionExpress() {
+        return conditionExpress;
+    }
+
+    public boolean isSkipJDKTrace() {
+        return skipJDKTrace;
+    }
+
+    public boolean isRegEx() {
+        return isRegEx;
+    }
+
+    public int getNumberOfLimit() {
+        return numberOfLimit;
+    }
+
+    public List<String> getPathPatterns() {
+        return pathPatterns;
+    }
+
+    @Override
+    protected Matcher getClassNameMatcher() {
+        if (classNameMatcher == null) {
+            if (pathPatterns == null || pathPatterns.isEmpty()) {
+                classNameMatcher = SearchUtils.classNameMatcher(getClassPattern(), isRegEx());
+            } else {
+                classNameMatcher = getPathTracingClassMatcher();
+            }
+        }
+        return classNameMatcher;
+    }
+
+    @Override
+    protected Matcher getClassNameExcludeMatcher() {
+        if (classNameExcludeMatcher == null && getExcludeClassPattern() != null) {
+            classNameExcludeMatcher = SearchUtils.classNameMatcher(getExcludeClassPattern(), isRegEx());
+        }
+        return classNameExcludeMatcher;
+    }
+
+    @Override
+    protected Matcher getMethodNameMatcher() {
+        if (methodNameMatcher == null) {
+            if (pathPatterns == null || pathPatterns.isEmpty()) {
+                methodNameMatcher = SearchUtils.classNameMatcher(getMethodPattern(), isRegEx());
+            } else {
+                methodNameMatcher = getPathTracingMethodMatcher();
+            }
+        }
+        return methodNameMatcher;
+    }
+
+    @Override
+    protected AdviceListener getAdviceListener(CommandProcess process) {
+        if (pathPatterns == null || pathPatterns.isEmpty()) {
+            return new TraceAdviceListener(this, process, GlobalOptions.verbose || this.verbose);
+        } else {
+            return new PathTraceAdviceListener(this, process);
+        }
+    }
+
+    /**
+     * 构造追踪路径匹配
+     */
+    private Matcher<String> getPathTracingClassMatcher() {
+
+        List<Matcher<String>> matcherList = new ArrayList<Matcher<String>>();
+        matcherList.add(SearchUtils.classNameMatcher(getClassPattern(), isRegEx()));
+
+        if (null != getPathPatterns()) {
+            for (String pathPattern : getPathPatterns()) {
+                if (isRegEx()) {
+                    matcherList.add(new RegexMatcher(pathPattern));
+                } else {
+                    matcherList.add(new WildcardMatcher(pathPattern));
+                }
+            }
+        }
+
+        return new GroupMatcher.Or<String>(matcherList);
+    }
+
+    private Matcher<String> getPathTracingMethodMatcher() {
+        return new TrueMatcher<String>();
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/TraceEntity.java

+package com.taobao.arthas.core.command.monitor200;
+
+import com.taobao.arthas.core.command.model.TraceModel;
+import com.taobao.arthas.core.command.model.TraceTree;
+import com.taobao.arthas.core.util.ThreadUtil;
+
+/**
+ * 用于在ThreadLocal中传递的实体
+ * @author ralf0131 2017-01-05 14:05.
+ */
+public class TraceEntity {
+
+    protected TraceTree tree;
+    protected int deep;
+
+    public TraceEntity(ClassLoader loader) {
+        this.tree = createTraceTree(loader);
+        this.deep = 0;
+    }
+
+    private TraceTree createTraceTree(ClassLoader loader) {
+        return new TraceTree(ThreadUtil.getThreadNode(loader, Thread.currentThread()));
+    }
+
+    public TraceModel getModel() {
+        tree.trim();
+        return new TraceModel(tree.getRoot(), tree.getNodeCount());
+    }
+}

core/src/main/java/com/taobao/arthas/core/command/monitor200/VmToolCommand.java

+package com.taobao.arthas.core.command.monitor200;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.lang.instrument.Instrumentation;
+import java.security.CodeSource;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.alibaba.arthas.deps.org.slf4j.Logger;
+import com.alibaba.arthas.deps.org.slf4j.LoggerFactory;
+import com.taobao.arthas.common.IOUtils;
+import com.taobao.arthas.common.VmToolUtils;
+import com.taobao.arthas.core.command.Constants;
+import com.taobao.arthas.core.command.express.Express;
+import com.taobao.arthas.core.command.express.ExpressException;
+import com.taobao.arthas.core.command.express.ExpressFactory;
+import com.taobao.arthas.core.command.model.ClassLoaderVO;
+import com.taobao.arthas.core.command.model.ObjectVO;
+import com.taobao.arthas.core.command.model.VmToolModel;
+import com.taobao.arthas.core.shell.cli.Completion;
+import com.taobao.arthas.core.shell.cli.CompletionUtils;
+import com.taobao.arthas.core.shell.cli.OptionCompleteHandler;
+import com.taobao.arthas.core.shell.command.AnnotatedCommand;
+import com.taobao.arthas.core.shell.command.CommandProcess;
+import com.taobao.arthas.core.util.ClassLoaderUtils;
+import com.taobao.arthas.core.util.ClassUtils;
+import com.taobao.arthas.core.util.SearchUtils;
+import com.taobao.middleware.cli.annotations.DefaultValue;
+import com.taobao.middleware.cli.annotations.Description;
+import com.taobao.middleware.cli.annotations.Name;
+import com.taobao.middleware.cli.annotations.Option;
+import com.taobao.middleware.cli.annotations.Summary;
+
+import arthas.VmTool;
+
+/**
+ * 
+ * @author hengyunabc 2021-04-27
+ * @author ZhangZiCheng 2021-04-29
+ *
+ */
+//@formatter:off
+@Name("vmtool")
+@Summary("jvm tool")
+@Description(Constants.EXAMPLE
+        + "  vmtool --action getInstances --className demo.MathGame\n"
+        + "  vmtool --action getInstances --className demo.MathGame --express 'instances.length'\n"
+        + "  vmtool --action getInstances --className demo.MathGame --express 'instances[0]'\n"
+        + "  vmtool --action getInstances --className demo.MathGame -x 2\n"
+        + "  vmtool --action getInstances --className java.lang.String --limit 10\n"
+        + "  vmtool --action getInstances --classLoaderClass org.springframework.boot.loader.LaunchedURLClassLoader --className org.springframework.context.ApplicationContext\n"
+        + "  vmtool --action forceGc\n"
+        + Constants.WIKI + Constants.WIKI_HOME + "vmtool")
+//@formatter:on
+public class VmToolCommand extends AnnotatedCommand {
+    private static final Logger logger = LoggerFactory.getLogger(VmToolCommand.class);
+
+    private VmToolAction action;
+    private String className;
+    private String express;
+
+    private String hashCode = null;
+    private String classLoaderClass;
+    /**
+     * default value 1
+     */
+    private int expand;
+
+    /**
+     * default value 10
+     */
+    private int limit;
+
+    private String libPath;
+    private static String defaultLibPath;
+    private static VmTool vmTool = null;
+
+    static {
+        String libName = VmToolUtils.detectLibName();
+        if (libName != null) {
+            CodeSource codeSource = VmToolCommand.class.getProtectionDomain().getCodeSource();
+            if (codeSource != null) {
+                try {
+                    File bootJarPath = new File(codeSource.getLocation().toURI().getSchemeSpecificPart());
+                    File soFile = new File(bootJarPath.getParentFile(), "lib" + File.separator + libName);
+                    if (soFile.exists()) {
+                        defaultLibPath = soFile.getAbsolutePath();
+                    }
+                } catch (Throwable e) {
+                    logger.error("can not find VmTool so", e);
+                }
+            }
+        }
+
+    }
+
+    @Option(shortName = "a", longName = "action", required = true)
+    @Description("Action to execute")
+    public void setAction(VmToolAction action) {
+        this.action = action;
+    }
+
+    @Option(longName = "className")
+    @Description("The class name")
+    public void setClassName(String className) {
+        this.className = className;
+    }
+
+    @Option(shortName = "x", longName = "expand")
+    @Description("Expand level of object (1 by default)")
+    @DefaultValue("1")
+    public void setExpand(int expand) {
+        this.expand = expand;
+    }
+
+    @Option(shortName = "c", longName = "classloader")
+    @Description("The hash code of the special class's classLoader")
+    public void setHashCode(String hashCode) {
+        this.hashCode = hashCode;
+    }
+
+    @Option(longName = "classLoaderClass")
+    @Description("The class name of the special class's classLoader.")
+    public void setClassLoaderClass(String classLoaderClass) {
+        this.classLoaderClass = classLoaderClass;
+    }
+
+    @Option(shortName = "l", longName = "limit")
+    @Description("Set the limit value of the getInstances action, default value is 10, set to -1 is unlimited")
+    @DefaultValue("10")
+    public void setLimit(int limit) {
+        this.limit = limit;
+    }
+
+    @Option(longName = "libPath")
+    @Description("The specify lib path.")
+    public void setLibPath(String path) {
+        libPath = path;
+    }
+
+    @Option(longName = "express", required = false)
+    @Description("The ognl expression, default valueis `instances`.")
+    public void setExpress(String express) {
+        this.express = express;
+    }
+
+    public enum VmToolAction {
+        getInstances, forceGc
+    }
+
+    @Override
+    public void process(final CommandProcess process) {
+        try {
+            Instrumentation inst = process.session().getInstrumentation();
+
+            if (VmToolAction.getInstances.equals(action)) {
+                if (className == null) {
+                    process.end(-1, "The className option cannot be empty!");
+                    return;
+                }
+                ClassLoader classLoader = null;
+                if (hashCode != null) {
+                    classLoader = ClassLoaderUtils.getClassLoader(inst, hashCode);
+                    if (classLoader == null) {
+                        process.end(-1, "Can not find classloader with hashCode: " + hashCode + ".");
+                        return;
+                    }
+                }else if ( classLoaderClass != null) {
+                    List<ClassLoader> matchedClassLoaders = ClassLoaderUtils.getClassLoaderByClassName(inst,
+                            classLoaderClass);
+                    if (matchedClassLoaders.size() == 1) {
+                        classLoader = matchedClassLoaders.get(0);
+                        hashCode = Integer.toHexString(matchedClassLoaders.get(0).hashCode());
+                    } else if (matchedClassLoaders.size() > 1) {
+                        Collection<ClassLoaderVO> classLoaderVOList = ClassUtils
+                                .createClassLoaderVOList(matchedClassLoaders);
+
+                        VmToolModel vmToolModel = new VmToolModel().setClassLoaderClass(classLoaderClass)
+                                .setMatchedClassLoaders(classLoaderVOList);
+                        process.appendResult(vmToolModel);
+                        process.end(-1,
+                                "Found more than one classloader by class name, please specify classloader with '-c <classloader hash>'");
+                        return;
+                    } else {
+                        process.end(-1, "Can not find classloader by class name: " + classLoaderClass + ".");
+                        return;
+                    }
+                }else {
+                    classLoader = ClassLoader.getSystemClassLoader();
+                }
+
+                List<Class<?>> matchedClasses = new ArrayList<Class<?>>(
+                        SearchUtils.searchClassOnly(inst, className, false, hashCode));
+                int matchedClassSize = matchedClasses.size();
+                if (matchedClassSize == 0) {
+                    process.end(-1, "Can not find class by class name: " + className + ".");
+                    return;
+                } else if (matchedClassSize > 1) {
+                    process.end(-1, "Found more than one class: " + matchedClasses + ", please specify classloader with '-c <classloader hash>'");
+                    return;
+                } else {
+                    Object[] instances = vmToolInstance().getInstances(matchedClasses.get(0), limit);
+                    Object value = instances;
+                    if (express != null) {
+                        Express unpooledExpress = ExpressFactory.unpooledExpress(classLoader);
+                        try {
+                            value = unpooledExpress.bind(new InstancesWrapper(instances)).get(express);
+                        } catch (ExpressException e) {
+                            logger.warn("ognl: failed execute express: " + express, e);
+                            process.end(-1, "Failed to execute ognl, exception message: " + e.getMessage()
+                                    + ", please check $HOME/logs/arthas/arthas.log for more details. ");
+                        }
+                    }
+
+                    VmToolModel vmToolModel = new VmToolModel().setValue(new ObjectVO(value, expand));
+                    process.appendResult(vmToolModel);
+                    process.end();
+                }
+            } else if (VmToolAction.forceGc.equals(action)) {
+                vmToolInstance().forceGc();
+                process.write("\n");
+                process.end();
+                return;
+            }
+
+            process.end();
+        } catch (Throwable e) {
+            logger.error("vmtool error", e);
+            process.end(1, "vmtool error: " + e.getMessage());
+        }
+    }
+
+    static class InstancesWrapper {
+        Object instances;
+
+        public InstancesWrapper(Object instances) {
+            this.instances = instances;
+        }
+
+        public Object getInstances() {
+            return instances;
+        }
+
+        public void setInstances(Object instances) {
+            this.instances = instances;
+        }
+    }
+
+    private VmTool vmToolInstance() {
+        if (vmTool != null) {
+            return vmTool;
+        } else {
+            if (libPath == null) {
+                libPath = defaultLibPath;
+            }
+
+            // 尝试把lib文件复制到临时文件里，避免多次attach时出现 Native Library already loaded in another classloader
+            FileOutputStream tmpLibOutputStream = null;
+            FileInputStream libInputStream = null;
+            try {
+                File tmpLibFile = File.createTempFile(VmTool.JNI_LIBRARY_NAME, null);
+                tmpLibOutputStream = new FileOutputStream(tmpLibFile);
+                libInputStream = new FileInputStream(libPath);
+
+                IOUtils.copy(libInputStream, tmpLibOutputStream);
+                libPath = tmpLibFile.getAbsolutePath();
+                logger.debug("copy {} to {}", libPath, tmpLibFile);
+            } catch (Throwable e) {
+                logger.error("try to copy lib error! libPath: {}", libPath, e);
+            } finally {
+                IOUtils.close(libInputStream);
+                IOUtils.close(tmpLibOutputStream);
+            }
+
+            vmTool = VmTool.getInstance(libPath);
+        }
+        return vmTool;
+    }
+
+    private Set<String> actions() {
+        Set<String> values = new HashSet<String>();
+        for (VmToolAction action : VmToolAction.values()) {
+            values.add(action.toString());
+        }
+        return values;
+    }
+
+    @Override
+    public void complete(Completion completion) {
+        List<OptionCompleteHandler> handlers = new ArrayList<OptionCompleteHandler>();
+
+        handlers.add(new OptionCompleteHandler() {
+
+            @Override
+            public boolean matchName(String token) {
+                return "-a".equals(token) || "--action".equals(token);
+            }
+
+            @Override
+            public boolean complete(Completion completion) {
+                return CompletionUtils.complete(completion, actions());
+            }
+
+        });
+
+        handlers.add(new OptionCompleteHandler() {
+            @Override
+            public boolean matchName(String token) {
+                return "--className".equals(token);
+            }
+
+            @Override
+            public boolean complete(Completion completion) {
+                return CompletionUtils.completeClassName(completion);
+            }
+        });
+
+        if (CompletionUtils.completeOptions(completion, handlers)) {
+            return;
+        }
+
+        super.complete(completion);
+    }
+
+}