记一次生产频繁发生FullGC问题

Posted on by hackdl

 T asyncWithTimeout(ScheduledThreadPoolExecutor executor, Callable callable,”,”marks”:[]}]}]},{“type”:”block”,”id”:”ANNW-1678943849392″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”23vq-1678943849391″,”leaves”:[{“text”:”                              long time, TimeUnit unit) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”hBX8-1678943849394″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Ex7q-1678943849393″,”leaves”:[{“text”:”    try {“,”marks”:[]}]}]},{“type”:”block”,”id”:”aLNc-1678943849396″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”wGry-1678943849395″,”leaves”:[{“text”:”        ListeningExecutorService listeningExecutorService = MoreExecutors.listeningDecorator(threadPoolExecutor);”,”marks”:[]}]}]},{“type”:”block”,”id”:”pD9s-1678943849398″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”MNi1-1678943849397″,”leaves”:[{“text”:”        ListenableFuture future = listeningExecutorService.submit(callable);”,”marks”:[]}]}]},{“type”:”block”,”id”:”NNba-1678943937650″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”9juk-1678943937648″,”leaves”:[{“text”:”        //这里是创建一个定时任务线程，去定时检查是否在规定时间内查询完毕，应该就是这个去添加了钩子函数，进去看看”,”marks”:[]}]}]},{“type”:”block”,”id”:”opZ9-1678943849400″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”WOUd-1678943849399″,”leaves”:[{“text”:”        ScheduledExecutorService scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(executor);”,”marks”:[]}]}]},{“type”:”block”,”id”:”oOgk-1678943849403″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”07kH-1678943849402″,”leaves”:[{“text”:”        return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit);”,”marks”:[]}]}]},{“type”:”block”,”id”:”PPTm-1678943849405″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”QHpI-1678943849404″,”leaves”:[{“text”:”    } catch (InterruptedException | ExecutionException | TimeoutException e) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”gNcQ-1678943849407″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”EDcW-1678943849406″,”leaves”:[{“text”:”        log.warn(“异步方法执行失败，error:{}”, e.getMessage());”,”marks”:[]}]}]},{“type”:”block”,”id”:”6SaZ-1678943849409″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”tst6-1678943849408″,”leaves”:[{“text”:”    }”,”marks”:[]}]}]},{“type”:”block”,”id”:”REWn-1678943849411″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”eJpf-1678943849410″,”leaves”:[{“text”:”    return null;”,”marks”:[]}]}]},{“type”:”block”,”id”:”iYw0-1678943849413″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”bKCg-1678943849412″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”jFI0-1678944017333″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”yqTo-1678944017332″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”7pCC-1678944036664″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”5dwS-1678944036663″,”leaves”:[{“text”:”//=======================guava并发包代码=======================”,”marks”:[]}]}]},{“type”:”block”,”id”:”a2RD-1678944035616″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”O65x-1678944035614″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”V0Qx-1678944017491″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”NyBg-1678944017490″,”leaves”:[{“text”:”@Beta”,”marks”:[]}]}]},{“type”:”block”,”id”:”FZ8L-1678944032991″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”kdmS-1678944032990″,”leaves”:[{“text”:”@GwtIncompatible // TODO”,”marks”:[]}]}]},{“type”:”block”,”id”:”JNy8-1678944032993″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”VgFq-1678944032992″,”leaves”:[{“text”:”public static ScheduledExecutorService getExitingScheduledExecutorService(“,”marks”:[]}]}]},{“type”:”block”,”id”:”sShc-1678944032995″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”LZMn-1678944032994″,”leaves”:[{“text”:”    ScheduledThreadPoolExecutor executor) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”PC5k-1678944111319″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”3GyA-1678944111317″,”leaves”:[{“text”:”  //每次都去创建一个新的对象”,”marks”:[]}]}]},{“type”:”block”,”id”:”IwXQ-1678944032997″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”qzIj-1678944032996″,”leaves”:[{“text”:”  return new Application().getExitingScheduledExecutorService(executor);”,”marks”:[]}]}]},{“type”:”block”,”id”:”4khx-1678944032999″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”0tC7-1678944032998″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”DumM-1678944063860″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”pD8Y-1678944063859″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”EE8E-1678944064031″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”cntO-1678944064029″,”leaves”:[{“text”:”final ScheduledExecutorService getExitingScheduledExecutorService(“,”marks”:[]}]}]},{“type”:”block”,”id”:”dBO9-1678944080489″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Zd8n-1678944080488″,”leaves”:[{“text”:”    ScheduledThreadPoolExecutor executor) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”2N4D-1678944080491″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”HkL9-1678944080490″,”leaves”:[{“text”:”  return getExitingScheduledExecutorService(executor, 120, TimeUnit.SECONDS);”,”marks”:[]}]}]},{“type”:”block”,”id”:”z00j-1678944080493″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Qucd-1678944080492″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”iDcV-1678944091423″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”ervU-1678944091422″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”zFCO-1678944091557″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”DEiW-1678944091555″,”leaves”:[{“text”:”final ScheduledExecutorService getExitingScheduledExecutorService(“,”marks”:[]}]}]},{“type”:”block”,”id”:”vtdW-1678944103053″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”cMNM-1678944103052″,”leaves”:[{“text”:”    ScheduledThreadPoolExecutor executor, long terminationTimeout, TimeUnit timeUnit) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”8YYl-1678944103055″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”q2lF-1678944103054″,”leaves”:[{“text”:”  useDaemonThreadFactory(executor);”,”marks”:[]}]}]},{“type”:”block”,”id”:”PLSW-1678944103057″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”4Agj-1678944103056″,”leaves”:[{“text”:”  ScheduledExecutorService service = Executors.unconfigurableScheduledExecutorService(executor);”,”marks”:[]}]}]},{“type”:”block”,”id”:”Zbhg-1678944170965″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”W2yk-1678944170963″,”leaves”:[{“text”:”  //添加构造函数的地方，进去看看”,”marks”:[]}]}]},{“type”:”block”,”id”:”No0d-1678944103059″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Rfk1-1678944103058″,”leaves”:[{“text”:”  addDelayedShutdownHook(executor, terminationTimeout, timeUnit);”,”marks”:[]}]}]},{“type”:”block”,”id”:”Fp48-1678944103061″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mYgx-1678944103060″,”leaves”:[{“text”:”  return service;”,”marks”:[]}]}]},{“type”:”block”,”id”:”xLR9-1678944103063″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”HJoq-1678944103062″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”twL7-1678944186649″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”WfPq-1678944186647″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”wp2T-1678944187010″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”0wBM-1678944187009″,”leaves”:[{“text”:”final void addDelayedShutdownHook(“,”marks”:[]}]}]},{“type”:”block”,”id”:”3zVZ-1678944209261″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Gcw
X-1678944209260″,”leaves”:[{“text”:”    final ExecutorService service, final long terminationTimeout, final TimeUnit timeUnit) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”dC4T-1678944209263″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”DDDc-1678944209262″,”leaves”:[{“text”:”  checkNotNull(service);”,”marks”:[]}]}]},{“type”:”block”,”id”:”gd9n-1678944209265″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”7tWB-1678944209264″,”leaves”:[{“text”:”  checkNotNull(timeUnit);”,”marks”:[]}]}]},{“type”:”block”,”id”:”49gN-1678944263886″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”xef9-1678944263885″,”leaves”:[{“text”:”  //继续点进去”,”marks”:[]}]}]},{“type”:”block”,”id”:”IIkZ-1678944209267″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”iYRD-1678944209266″,”leaves”:[{“text”:”  addShutdownHook(“,”marks”:[]}]}]},{“type”:”block”,”id”:”vl9s-1678944209269″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”RfWS-1678944209268″,”leaves”:[{“text”:”      MoreExecutors.newThread(“,”marks”:[]}]}]},{“type”:”block”,”id”:”uqGl-1678944234121″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”kmnZ-1678944234120″,”leaves”:[{“text”:”          //线程名字对上了，就在对象引用的截图里面出现过”,”marks”:[]}]}]},{“type”:”block”,”id”:”8Y5K-1678944209271″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”i1Zx-1678944209270″,”leaves”:[{“text”:”          “DelayedShutdownHook-for-” + service,”,”marks”:[]}]}]},{“type”:”block”,”id”:”jqM7-1678944209273″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”o0hw-1678944209272″,”leaves”:[{“text”:”          new Runnable() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”Pgl3-1678944209275″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”JPwu-1678944209274″,”leaves”:[{“text”:”            @Override”,”marks”:[]}]}]},{“type”:”block”,”id”:”anEQ-1678944209277″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”i8vC-1678944209276″,”leaves”:[{“text”:”            public void run() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”t7cv-1678944209279″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”RLLH-1678944209278″,”leaves”:[{“text”:”              try {“,”marks”:[]}]}]},{“type”:”block”,”id”:”sVFu-1678944209281″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”iwxX-1678944209280″,”leaves”:[{“text”:”                // We’d like to log progress and failures that may arise in the”,”marks”:[]}]}]},{“type”:”block”,”id”:”bg4I-1678944209283″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”I3Qj-1678944209282″,”leaves”:[{“text”:”                // following code, but unfortunately the behavior of logging”,”marks”:[]}]}]},{“type”:”block”,”id”:”7sqz-1678944209285″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”vioK-1678944209284″,”leaves”:[{“text”:”                // is undefined in shutdown hooks.”,”marks”:[]}]}]},{“type”:”block”,”id”:”5MfL-1678944209287″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”EmOb-1678944209286″,”leaves”:[{“text”:”                // This is because the logging code installs a shutdown hook of its”,”marks”:[]}]}]},{“type”:”block”,”id”:”zE77-1678944209289″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”d8H3-1678944209288″,”leaves”:[{“text”:”                // own. See Cleaner class inside {@link LogManager}.”,”marks”:[]}]}]},{“type”:”block”,”id”:”1Vzx-1678944209291″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”GXGr-1678944209290″,”leaves”:[{“text”:”                service.shutdown();”,”marks”:[]}]}]},{“type”:”block”,”id”:”imIN-1678944209293″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”lXql-1678944209292″,”leaves”:[{“text”:”                service.awaitTermination(terminationTimeout, timeUnit);”,”marks”:[]}]}]},{“type”:”block”,”id”:”6EYH-1678944209295″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”f6Lo-1678944209294″,”leaves”:[{“text”:”              } catch (InterruptedException ignored) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”gRfG-1678944209297″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”WNV7-1678944209296″,”leaves”:[{“text”:”                // We’re shutting down anyway, so just ignore.”,”marks”:[]}]}]},{“type”:”block”,”id”:”KTZB-1678944209299″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”yXrN-1678944209298″,”leaves”:[{“text”:”              }”,”marks”:[]}]}]},{“type”:”block”,”id”:”quQo-1678944209302″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mqlD-1678944209301″,”leaves”:[{“text”:”            }”,”marks”:[]}]}]},{“type”:”block”,”id”:”bPVw-1678944209304″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”N61s-1678944209303″,”leaves”:[{“text”:”          }));”,”marks”:[]}]}]},{“type”:”block”,”id”:”TveW-1678944209306″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”kPpl-1678944209305″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”kbn5-1678944272787″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”UeRs-1678944272785″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”0cJ4-1678944272961″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”TXyH-1678944272959″,”leaves”:[{“text”:”@VisibleForTesting”,”marks”:[]}]}]},{“type”:”block”,”id”:”x9SR-1678944291440″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”UwN8-1678944291439″,”leaves”:[{“text”:”void addShutdownHook(Thread hook) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”SG6i-1678944291442″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”fqJS-1678944291441″,”leaves”:[{“text”:”  Runtime.getRuntime().addShutdownHook(hook);”,”marks”:[]}]}]},{“type”:”block”,”id”:”t3zv-1678944291444″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mzfV-1678944291443″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”TEJZ-1678945273665″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”GDGh-1678945273664″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”KqYc-1678945273812″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mvod-1678945273811″,”leaves”:[{“text”:”//=======================guava并发包代码=======================”,”marks”:[]}]}]},{“type”:”block”,”id”:”gQqi-1678944293317″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”99nH-1678944293316″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”asqG-1678944293442″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Vnje-1678944293441″,”leaves”:[{“text”:”public void addShutdownHook(Thread hook) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”DDSJ-1678944307824″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”QNFZ-1678944307823″,”leaves”:[{“text”:”    SecurityManager sm = System.getSecurityManager();”,”marks”:[]}]}]},{“type”:”block”,”id”:”GxQV-1678944307826″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”4haL-1678944307825″,”leaves”:[{“text”:”    if (sm != null) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”nMyO-1678944307828″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”kZ8Q-1678944307827″,”leaves”:[{“text”:”        sm.checkPermission(new RuntimePermission(“shutdownHooks”));”,”marks”:[]}]}]},{“type”:”block”,”id”:”hPPF-1678944307830″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”qjQs-1678944307829″,”leaves”:[{“text”:”    }”,”marks”:[]}]}]},{“type”:”block”,”id”:”lDC1-1678944309779″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”dCXz-1678944309777″,”leaves”:[{“text”:”    //定位到问题了，就是这里添加的钩子函数”,”marks”:[]}]}]},{“type”:”block”,”id”:”BUH1-1678944307832″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Q7tN-1678944307831″,”leaves”:[{“text”:”    ApplicationShutdownHooks.add(hook);”,”marks”:[]}]}]},{“type”:”block”,”id”:”Qwp1-1678944307834″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:
“text”,”id”:”J8Ba-1678944307833″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”W4D0-1678944346507″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”KNiT-1678944346505″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”7T9f-1678944346638″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Lbnd-1678944346637″,”leaves”:[{“text”:”static synchronized void add(Thread hook) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”VSfo-1678944346892″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”s8tQ-1678944346891″,”leaves”:[{“text”:”    if(hooks == null)”,”marks”:[]}]}]},{“type”:”block”,”id”:”vh4Q-1678944346894″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”o4Od-1678944346893″,”leaves”:[{“text”:”        throw new IllegalStateException(“Shutdown in progress”);”,”marks”:[]}]}]},{“type”:”block”,”id”:”IYJG-1678944346896″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”1S1U-1678944346895″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”lgFM-1678944346898″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”S96h-1678944346897″,”leaves”:[{“text”:”    if (hook.isAlive())”,”marks”:[]}]}]},{“type”:”block”,”id”:”3k2U-1678944346900″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”7dNA-1678944346899″,”leaves”:[{“text”:”        throw new IllegalArgumentException(“Hook already running”);”,”marks”:[]}]}]},{“type”:”block”,”id”:”tMc5-1678944346902″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”xP8g-1678944346901″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”ezlW-1678944346904″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”XahK-1678944346903″,”leaves”:[{“text”:”    if (hooks.containsKey(hook))”,”marks”:[]}]}]},{“type”:”block”,”id”:”Q79l-1678944346906″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”ctmw-1678944346905″,”leaves”:[{“text”:”        throw new IllegalArgumentException(“Hook previously registered”);”,”marks”:[]}]}]},{“type”:”block”,”id”:”XRNn-1678944351349″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”NnTP-1678944351348″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”eVH0-1678944352697″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Wxvy-1678944352696″,”leaves”:[{“text”:”    //存在到 hooks 这个map对象里面，就是这个大对象”,”marks”:[]}]}]},{“type”:”block”,”id”:”suuD-1678944346910″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”W47f-1678944346909″,”leaves”:[{“text”:”    hooks.put(hook, hook);”,”marks”:[]}]}]},{“type”:”block”,”id”:”5dAf-1678944346912″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”9YvD-1678944346911″,”leaves”:[{“text”:”}”,”marks”:[]}]}]}],”state”:{}},{“type”:”block”,”id”:”D4QM-1678943790878″,”name”:”paragraph”,”data”:{},”nodes”:[{“type”:”text”,”id”:”BlWQ-1678943790877″,”leaves”:[{“text”:””,”marks”:[]}]}],”state”:{}},{“type”:”block”,”id”:”JXMT-1678944427097″,”name”:”heading”,”data”:{“level”:”h2″,”style”:{}},”nodes”:[{“type”:”text”,”id”:”hoMo-1678944427096″,”leaves”:[{“text”:”问题解决”,”marks”:[]}]}],”state”:{}},{“type”:”block”,”id”:”ThGx-1678944431569″,”name”:”paragraph”,”data”:{“version”:1},”nodes”:[{“type”:”text”,”id”:”El0r-1678944431570″,”leaves”:[{“text”:”经过上面问题的排查，造成hooks大对象的原因找到了，就是每次调用接口的时候，都会往hooks里面put一个对象。”,”marks”:[]}]}],”state”:{}},{“type”:”block”,”id”:”JzJr-1678944576035″,”name”:”paragraph”,”data”:{“version”:1},”nodes”:[{“type”:”text”,”id”:”GVhs-1678944576033″,”leaves”:[{“text”:”所以，解决办法很简单，就是不用每次都去生成一个ScheduledExecutorService对象，类初始化的时候创建一次就行了”,”marks”:[]}]}],”state”:{}},{“type”:”block”,”id”:”3I06-1678944655450″,”name”:”paragraph”,”data”:{“version”:1},”nodes”:[{“type”:”text”,”id”:”yQvH-1678944655449″,”leaves”:[{“text”:”改造后的代码如下：”,”marks”:[]}]}],”state”:{}},{“type”:”block”,”id”:”k4bH-1678944673397″,”name”:”code”,”data”:{“language”:”java”,”theme”:”default”,”wrap”:false},”nodes”:[{“type”:”block”,”id”:”CdsV-1678944673396″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”VfTL-1678944670183″,”leaves”:[{“text”:”private ListeningExecutorService listeningExecutorService;”,”marks”:[]}]}]},{“type”:”block”,”id”:”qmsp-1678945157919″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”spP6-1678945157918″,”leaves”:[{“text”:”private ScheduledExecutorService scheduledExecutorService;”,”marks”:[]}]}]},{“type”:”block”,”id”:”ROLU-1678945156955″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”X79W-1678945156953″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”DZOI-1678945225297″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”rOtl-1678945225296″,”leaves”:[{“text”:”public static AsyncUtils getInstance() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”ustS-1678945225692″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Wv2A-1678945225691″,”leaves”:[{“text”:”    return ThreadHolder.INSTANCE.getAsyncWithCallback();”,”marks”:[]}]}]},{“type”:”block”,”id”:”MTsd-1678945225694″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”3B8I-1678945225693″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”Nh3y-1678945224401″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Yg74-1678945224399″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”1xds-1678945157082″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”zGEz-1678945157081″,”leaves”:[{“text”:”@SuppressWarnings(“UnstableApiUsage”)”,”marks”:[]}]}]},{“type”:”block”,”id”:”9i4n-1678945148529″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”sryJ-1678945148528″,”leaves”:[{“text”:”private AsyncUtils() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”nUel-1678945148531″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”b3Ua-1678945148530″,”leaves”:[{“text”:”    listeningExecutorService = MoreExecutors.listeningDecorator(ThreadPoolConstant.THREAD_POOL_EXECUTOR);”,”marks”:[]}]}]},{“type”:”block”,”id”:”fCjF-1678945148533″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”40ci-1678945148532″,”leaves”:[{“text”:”    scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(ThreadPoolConstant.SCHEDULED_THREAD_POOL_EXECUTOR);”,”marks”:[]}]}]},{“type”:”block”,”id”:”0RSA-1678945148535″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”QKrr-1678945148534″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”NGva-1678945147204″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Ijw2-1678945147203″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”nSFF-1678945147342″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”0mUG-1678945147341″,”leaves”:[{“text”:”@SuppressWarnings(“UnstableApiUsage”)”,”marks”:[]}]}]},{“type”:”block”,”id”:”zT5O-1678945126795″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”cITm-1678945126794″,”leaves”:[{“text”:”public  T asyncWithTimeout(Callable callable,”,”marks”:[]}]}]},{“type”:”block”,”id”:”7zA4-1678945126797″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mB4s-1678945126796″,”leaves”:[{“text”:”                              long time, TimeUnit unit) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”XYPZ-1678945126799″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Xtld-1678945126798″,”leaves”:[{“text”:”    try {“,”marks”:[]}]}]},{“type”:”block”,”id”:”tBSS-1678945126801″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”JuK5-1678945126800″,”leaves”:[{“text”:”        ListenableFuture future = listeningExecutorService.submit(callable);”,”marks”:[]}]}]},{“type”:”block”,”id”:”Q6G7-1678945126803″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”9TLh-1678945126802″,”leaves”:[{“text”:”        return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit);”,”marks”:[]}]}]},{“type”:”block”,”id”:”B8Fr-1678945126805″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”FZTT-1678945126804″,”lea
ves”:[{“text”:”    } catch (InterruptedException | ExecutionException | TimeoutException e) {“,”marks”:[]}]}]},{“type”:”block”,”id”:”r318-1678945126807″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”bOFx-1678945126806″,”leaves”:[{“text”:”        log.warn(“异步方法执行失败，error:{}”, e.getMessage());”,”marks”:[]}]}]},{“type”:”block”,”id”:”69MW-1678945126809″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”66Fc-1678945126808″,”leaves”:[{“text”:”    }”,”marks”:[]}]}]},{“type”:”block”,”id”:”NZfH-1678945126811″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”ySnZ-1678945126810″,”leaves”:[{“text”:”    return null;”,”marks”:[]}]}]},{“type”:”block”,”id”:”qzyP-1678945126813″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”r2GP-1678945126812″,”leaves”:[{“text”:”}”,”marks”:[]}]}]},{“type”:”block”,”id”:”QwPd-1678945235026″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”Nixk-1678945235025″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”y9dw-1678945235163″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”2IPl-1678945235162″,”leaves”:[{“text”:”private enum ThreadHolder {“,”marks”:[]}]}]},{“type”:”block”,”id”:”Buta-1678945235464″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”A4wv-1678945235463″,”leaves”:[{“text”:”    /**”,”marks”:[]}]}]},{“type”:”block”,”id”:”yhqt-1678945235466″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”GuCX-1678945235465″,”leaves”:[{“text”:”     * 线程持有类 INSTANCE”,”marks”:[]}]}]},{“type”:”block”,”id”:”n7SS-1678945235468″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”mmH7-1678945235467″,”leaves”:[{“text”:”     */”,”marks”:[]}]}]},{“type”:”block”,”id”:”7s81-1678945235470″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”I7Sk-1678945235469″,”leaves”:[{“text”:”    INSTANCE;”,”marks”:[]}]}]},{“type”:”block”,”id”:”bEQp-1678945235472″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”N1Zt-1678945235471″,”leaves”:[{“text”:”    private final AsyncUtils asyncUtils;”,”marks”:[]}]}]},{“type”:”block”,”id”:”AczF-1678945235474″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”hFdo-1678945235473″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”pcBo-1678945235476″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”1Owz-1678945235475″,”leaves”:[{“text”:”    ThreadHolder() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”GmiP-1678945235478″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”2EO6-1678945235477″,”leaves”:[{“text”:”        asyncUtils = new AsyncUtils();”,”marks”:[]}]}]},{“type”:”block”,”id”:”eXLn-1678945235480″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”UCfi-1678945235479″,”leaves”:[{“text”:”    }”,”marks”:[]}]}]},{“type”:”block”,”id”:”IuJw-1678945235482″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”QF54-1678945235481″,”leaves”:[{“text”:””,”marks”:[]}]}]},{“type”:”block”,”id”:”CTBc-1678945235484″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”y3xG-1678945235483″,”leaves”:[{“text”:”    public AsyncUtils getAsyncWithCallback() {“,”marks”:[]}]}]},{“type”:”block”,”id”:”zKFN-1678945235486″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”qWIw-1678945235485″,”leaves”:[{“text”:”        return asyncUtils;”,”marks”:[]}]}]},{“type”:”block”,”id”:”5OJq-1678945235488″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”BEd5-1678945235487″,”leaves”:[{“text”:”    }”,”marks”:[]}]}]},{“type”:”block”,”id”:”opg3-1678945235490″,”name”:”code-line”,”data”:{},”nodes”:[{“type”:”text”,”id”:”MVv6-1678945235489″,”leaves”:[{“text”:”}”,”marks”:[]}]}]}],”state”:{}},{“type”:”block”,”id”:”yESz-1678932080903″,”name”:”paragraph”,”data”:{},”nodes”:[{“type”:”text”,”id”:”zoW5-1678932080902″,”leaves”:[{“text”:””,”marks”:[]}]}],”state”:{}}]”>

 

问题发现

早上过来，饭都没来的及吃，运维就给我发来信息，说是某个接口调用大量超时。因为最近这个接口调用量是翻倍了，所以我就去检查了下慢SQL，发现确实是有较多的慢SQL，所以我就缩减了查询的时间范围，但是效果并不好。
过了一会发现，这个服务fullGC是有问题的，太频繁了，这个应该是导致接口超时的根本问题，因为时间也是对的上的。
这个是最近三个小时fullGC的监控图：
这个是最近三天fullGC的监控图：
对比一下，就不难发现，fullGC数量是从3月15号晚上9点开始增加的，也是这个接口对外开放的时间。
 

解决思路

1、首先去服务器上面下载dump文件，分析是哪里造成了内存泄漏，频繁触发fullGC。首先找出服务器内java文件的PID，然后保存dump文件，我们公司java服务是固定端口号：1

使用top命令：
然后执行命令：jmap -dump:file=202303160924.dump,format=b 1 ，保存dump文件

2、根据dump文件，分析出堆内对象的分布情况

    • 下载一个可以分析dump文件的工具，这里我下载是Jprofiler
    • 查看大对象的分析，发现是java.lang.ApplicationShutdownHooks的hooks占用太大内存，并且得知改熟悉是一个Map
    • 分析这个Map里面的元素引用关系，可以看到这个map里面存的都是线程对象，并且大部分都是一个名为java.util.concurrent.ScheduledThreadPoolExecutor@59648a61的线程池对象，到了这里就定位到问题代码了，是这次新加的接口里面有一个异步操作，用的guava并发包里面的一个超时等待功能的接口，具体思路就是启用一个定时任务线程池去定时去检查在规定时间内，是否返回结果。
  

3、看看我的业务代码是哪里出现了问题

//异步执行某个查询方法，并且在规定时间内返回查询结果
public  T asyncWithTimeout(ScheduledThreadPoolExecutor executor, Callable callable,
                              long time, TimeUnit unit) {
    try {
        ListeningExecutorService listeningExecutorService = MoreExecutors.listeningDecorator(threadPoolExecutor);
        ListenableFuture future = listeningExecutorService.submit(callable);
        //这里是创建一个定时任务线程，去定时检查是否在规定时间内查询完毕，应该就是这个去添加了钩子函数，进去看看
        ScheduledExecutorService scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(executor);
        return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        log.warn("异步方法执行失败，error:{}", e.getMessage());
    }
    return null;
}

//=======================guava并发包代码=======================

@Beta
@GwtIncompatible // TODO
public static ScheduledExecutorService getExitingScheduledExecutorService(
    ScheduledThreadPoolExecutor executor) {
  //每次都去创建一个新的对象
  return new Application().getExitingScheduledExecutorService(executor);
}

final ScheduledExecutorService getExitingScheduledExecutorService(
    ScheduledThreadPoolExecutor executor) {
  return getExitingScheduledExecutorService(executor, 120, TimeUnit.SECONDS);
}

final ScheduledExecutorService getExitingScheduledExecutorService(
    ScheduledThreadPoolExecutor executor, long terminationTimeout, TimeUnit timeUnit) {
  useDaemonThreadFactory(executor);
  ScheduledExecutorService service = Executors.unconfigurableScheduledExecutorService(executor);
  //添加构造函数的地方，进去看看
  addDelayedShutdownHook(executor, terminationTimeout, timeUnit);
  return service;
}

final void addDelayedShutdownHook(
    final ExecutorService service, final long terminationTimeout, final TimeUnit timeUnit) {
  checkNotNull(service);
  checkNotNull(timeUnit);
  //继续点进去
  addShutdownHook(
      MoreExecutors.newThread(
          //线程名字对上了，就在对象引用的截图里面出现过
          "DelayedShutdownHook-for-" + service,
          new Runnable() {
            @Override
            public void run() {
              try {
                // We'd like to log progress and failures that may arise in the
                // following code, but unfortunately the behavior of logging
                // is undefined in shutdown hooks.
                // This is because the logging code installs a shutdown hook of its
                // own. See Cleaner class inside {@link LogManager}.
                service.shutdown();
                service.awaitTermination(terminationTimeout, timeUnit);
              } catch (InterruptedException ignored) {
                // We're shutting down anyway, so just ignore.
              }
            }
          }));
}

@VisibleForTesting
void addShutdownHook(Thread hook) {
  Runtime.getRuntime().addShutdownHook(hook);
}

//=======================guava并发包代码=======================

public void addShutdownHook(Thread hook) {
    SecurityManager sm = System.getSecurityManager();
    if (sm != null) {
        sm.checkPermission(new RuntimePermission("shutdownHooks"));
    }
    //定位到问题了，就是这里添加的钩子函数
    ApplicationShutdownHooks.add(hook);
}

static synchronized void add(Thread hook) {
    if(hooks == null)
        throw new IllegalStateException("Shutdown in progress");

    if (hook.isAlive())
        throw new IllegalArgumentException("Hook already running");

    if (hooks.containsKey(hook))
        throw new IllegalArgumentException("Hook previously registered");

    //存在到 hooks 这个map对象里面，就是这个大对象
    hooks.put(hook, hook);
}

问题解决

经过上面问题的排查，造成hooks大对象的原因找到了，就是每次调用接口的时候，都会往hooks里面put一个对象。
所以，解决办法很简单，就是不用每次都去生成一个ScheduledExecutorService对象，类初始化的时候创建一次就行了
改造后的代码如下：
private ListeningExecutorService listeningExecutorService;
private ScheduledExecutorService scheduledExecutorService;

public static AsyncUtils getInstance() {
    return ThreadHolder.INSTANCE.getAsyncWithCallback();
}

@SuppressWarnings("UnstableApiUsage")
private AsyncUtils() {
    listeningExecutorService = MoreExecutors.listeningDecorator(ThreadPoolConstant.THREAD_POOL_EXECUTOR);
    scheduledExecutorService = MoreExecutors.getExitingScheduledExecutorService(ThreadPoolConstant.SCHEDULED_THREAD_POOL_EXECUTOR);
}

@SuppressWarnings("UnstableApiUsage")
public  T asyncWithTimeout(Callable callable,
                              long time, TimeUnit unit) {
    try {
        ListenableFuture future = listeningExecutorService.submit(callable);
        return Futures.withTimeout(future, time, unit, scheduledExecutorService).get(time, unit);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        log.warn("异步方法执行失败，error:{}", e.getMessage());
    }
    return null;
}

private enum ThreadHolder {
    /**
     * 线程持有类 INSTANCE
     */
    INSTANCE;
    private final AsyncUtils asyncUtils;

    ThreadHolder() {
        asyncUtils = new AsyncUtils();
    }

    public AsyncUtils getAsyncWithCallback() {
        return asyncUtils;
    }
}
 

服务器托管，北京服务器托管，服务器租用 http://www.fwqtg.net
机房租用，北京机房租用，IDC机房托管， http://www.e1idc.net

服务器托管，北京服务器托管，服务器租用，机房机柜带宽租用

服务器托管

咨询：董先生

电话13051898268 QQ/微信93663045！

上一篇:
下一篇: