Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-46550

[CX] metadata lock deadlock between recovery & disconnect link

    XMLWordPrintable

Details

    • Untriaged
    • 1
    • Yes
    • CX Sprint 248

    Description

        {
          "entity": "ACTIVE:^Local:Local.testBucket",
          "locks": [
            {
              "lockField": "MetadataLock.lock",
              "lock": "java.util.concurrent.locks.ReentrantReadWriteLock$FairSync@6bfa4aa2",
              "firstReader": null,
              "writer": "Executor-50:ClusterController (tid=508)",
              "readLockCount": 0,
              "writeLockCount": 0,
              "queueLength": 1,
              "queuedThreads": [
                "QueryTranslator:0763e785-fbdf-443f-a5d7-de763d958b4d (tid=92)"
              ]
            }
          ]
        },
        {
          "entity": "EXTENSION:CB:testBucket/scope-0-2:Local",
          "locks": [
            {
              "lockField": "MetadataLock.lock",
              "lock": "java.util.concurrent.locks.ReentrantReadWriteLock$FairSync@7ec6308f",
              "firstReader": null,
              "writer": "QueryTranslator:0763e785-fbdf-443f-a5d7-de763d958b4d (tid=92)",
              "readLockCount": 0,
              "writeLockCount": 0,
              "queueLength": 1,
              "queuedThreads": [
                "Executor-50:ClusterController (tid=508)"
              ]
            }
          ]
        },
      

      "QueryTranslator:0763e785-fbdf-443f-a5d7-de763d958b4d" [tid=92 state=WAITING lock=java.util.concurrent.locks.ReentrantReadWriteLock$FairSync@6bfa4aa2 lockOwner="Executor-50:ClusterController" (tid=508]
      	at java.base@11.0.9.1/java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:959)
      	at app//org.apache.asterix.metadata.lock.MetadataLock.lock(MetadataLock.java:38)
      	at app//org.apache.asterix.common.metadata.LockList.add(LockList.java:52)
      	at app//org.apache.asterix.metadata.lock.MetadataLockManager.acquireActiveEntityWriteLock(MetadataLockManager.java:171)
      	at app//com.couchbase.analytics.metadata.CBMetadataLockUtil.acquireBucketEntityWriteLock(CBMetadataLockUtil.java:80)
      	at app//com.couchbase.analytics.metadata.CBMetadataLockUtil.lockBucket(CBMetadataLockUtil.java:73)
      	at app//com.couchbase.analytics.lang.ConnectionStatement.aggregateAndLockBuckets(ConnectionStatement.java:248)
      	at app//com.couchbase.analytics.lang.ConnectionStatement.generateConnectSpecs(ConnectionStatement.java:197)
      	at app//com.couchbase.analytics.lang.ConnectionStatement.prepareLinkConnectDisconnect(ConnectionStatement.java:181)
      	at app//com.couchbase.analytics.lang.DisconnectLinkStatement.doHandle(DisconnectLinkStatement.java:75)
      

      "Executor-50:ClusterController" [tid=508 state=WAITING lock=java.util.concurrent.locks.ReentrantReadWriteLock$FairSync@7ec6308f lockOwner="QueryTranslator:0763e785-fbdf-443f-a5d7-de763d958b4d" (tid=92]
      	at java.base@11.0.9.1/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
      	at app//org.apache.asterix.metadata.lock.MetadataLock.lock(MetadataLock.java:41)
      	at app//org.apache.asterix.common.metadata.LockList.add(LockList.java:52)
      	at app//org.apache.asterix.metadata.lock.MetadataLockManager.acquireExtensionEntityReadLock(MetadataLockManager.java:241)
      	at app//com.couchbase.analytics.metadata.CBMetadataLockUtil.acquireBucketEntityWriteLock(CBMetadataLockUtil.java:78)
      	at app//com.couchbase.analytics.metadata.CBMetadataLockUtil.lockBucket(CBMetadataLockUtil.java:73)
      	at app//com.couchbase.analytics.metadata.BucketEventsListener.lambda$persistBuckets$8(BucketEventsListener.java:353)
      	at app//com.couchbase.analytics.metadata.BucketEventsListener$$Lambda$863/0x0000000800983840.process(Unknown Source)
      	at app//org.apache.hyracks.util.ThrowingConsumer.lambda$forEach$2(ThrowingConsumer.java:54)
      	at app//org.apache.hyracks.util.ThrowingConsumer$$Lambda$862/0x0000000800984440.accept(Unknown Source)
      	at java.base@11.0.9.1/java.lang.Iterable.forEach(Iterable.java:75)
      	at app//org.apache.hyracks.util.ThrowingConsumer.forEach(ThrowingConsumer.java:52)
      	at app//com.couchbase.analytics.metadata.BucketEventsListener.persistBuckets(BucketEventsListener.java:352)
      	at app//com.couchbase.analytics.metadata.BucketEventsListener.persistBuckets(BucketEventsListener.java:342)
      	at app//com.couchbase.analytics.metadata.BucketEventsListener.setRunning(BucketEventsListener.java:337)
      	at app//org.apache.asterix.app.active.RecoveryTask.doRecover(RecoveryTask.java:167)
      

      Attachments

        1. cbcollect_info_n_0.zip
          11.60 MB
        2. cbcollect_info_n_1.zip
          6.46 MB
        3. cbcollect_info_n_2.zip
          8.19 MB
        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            umang.agrawal Umang
            michael.blow Michael Blow
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty