title: Kubernetes pod failed. Back off restarting failed container
body: <p>I have setup loki-grafana using helm to check logs. I have created it few days back and everything was working fine, but now suddenly started facing the error saying CrashLoopBackOff with error-code 2 in statefulset pod called loki-stack.Can anyone please help solving this issue?</p>
  <p>This is the last few logs of that pod</p>
  <pre><code>goroutine 183 [select]:
  github.com/grafana/loki/pkg/storage/chunk/cache.(*FifoCache).runPruneJob(0xc00066a8f0, 0x0, 0xc0004eb620)
      /src/loki/pkg/storage/chunk/cache/fifo_cache.go:213 +0xdf
  created by github.com/grafana/loki/pkg/storage/chunk/cache.NewFifoCache
      /src/loki/pkg/storage/chunk/cache/fifo_cache.go:202 +0x10f6
  
  goroutine 189 [select]:
  github.com/grafana/loki/pkg/storage/chunk/local.(*BoltIndexClient).loop(0xc00013f9f0)
      /src/loki/pkg/storage/chunk/local/boltdb_index_client.go:82 +0x105
  </code></pre>
  <p>this is deployment file</p>
  <pre><code>apiVersion: apps/v1
  kind: Deployment
  metadata:
    name: loki-stack-grafana
    namespace: grafana-loki
    uid: bf25a492-a0b2-40c2-9848-08931722e254
    resourceVersion: '145239013'
    generation: 4
    creationTimestamp: '2022-06-08T06:45:57Z'
    labels:
      app.kubernetes.io/instance: loki-stack
      app.kubernetes.io/managed-by: Helm
      app.kubernetes.io/name: grafana
      app.kubernetes.io/version: 8.3.5
      helm.sh/chart: grafana-6.24.1
    annotations:
      deployment.kubernetes.io/revision: '4'
      meta.helm.sh/release-name: loki-stack
      meta.helm.sh/release-namespace: grafana-loki
    managedFields:
      - manager: helm
        operation: Update
        apiVersion: apps/v1
        time: '2022-06-08T06:45:57Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:annotations:
              .: {}
              f:meta.helm.sh/release-name: {}
              f:meta.helm.sh/release-namespace: {}
            f:labels:
              .: {}
              f:app.kubernetes.io/instance: {}
              f:app.kubernetes.io/managed-by: {}
              f:app.kubernetes.io/name: {}
              f:app.kubernetes.io/version: {}
              f:helm.sh/chart: {}
          f:spec:
            f:progressDeadlineSeconds: {}
            f:replicas: {}
            f:revisionHistoryLimit: {}
            f:selector: {}
            f:strategy:
              f:rollingUpdate:
                .: {}
                f:maxSurge: {}
                f:maxUnavailable: {}
              f:type: {}
            f:template:
              f:metadata:
                f:annotations:
                  .: {}
                  f:checksum/config: {}
                  f:checksum/dashboards-json-config: {}
                  f:checksum/sc-dashboard-provider-config: {}
                  f:checksum/secret: {}
                f:labels:
                  .: {}
                  f:app.kubernetes.io/instance: {}
                  f:app.kubernetes.io/name: {}
              f:spec:
                f:automountServiceAccountToken: {}
                f:containers:
                  k:{&quot;name&quot;:&quot;grafana&quot;}:
                    .: {}
                    f:env:
                      .: {}
                      k:{&quot;name&quot;:&quot;GF_PATHS_DATA&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;GF_PATHS_LOGS&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;GF_PATHS_PLUGINS&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;GF_PATHS_PROVISIONING&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;GF_SECURITY_ADMIN_PASSWORD&quot;}:
                        .: {}
                        f:name: {}
                        f:valueFrom:
                          .: {}
                          f:secretKeyRef:
                            .: {}
                            f:key: {}
                            f:name: {}
                      k:{&quot;name&quot;:&quot;GF_SECURITY_ADMIN_USER&quot;}:
                        .: {}
                        f:name: {}
                        f:valueFrom:
                          .: {}
                          f:secretKeyRef:
                            .: {}
                            f:key: {}
                            f:name: {}
                    f:image: {}
                    f:imagePullPolicy: {}
                    f:livenessProbe:
                      .: {}
                      f:failureThreshold: {}
                      f:httpGet:
                        .: {}
                        f:path: {}
                        f:port: {}
                        f:scheme: {}
                      f:initialDelaySeconds: {}
                      f:periodSeconds: {}
                      f:successThreshold: {}
                      f:timeoutSeconds: {}
                    f:name: {}
                    f:ports:
                      .: {}
                      k:{&quot;containerPort&quot;:80,&quot;protocol&quot;:&quot;TCP&quot;}:
                        .: {}
                        f:containerPort: {}
                        f:name: {}
                        f:protocol: {}
                      k:{&quot;containerPort&quot;:3000,&quot;protocol&quot;:&quot;TCP&quot;}:
                        .: {}
                        f:containerPort: {}
                        f:name: {}
                        f:protocol: {}
                    f:readinessProbe:
                      .: {}
                      f:failureThreshold: {}
                      f:httpGet:
                        .: {}
                        f:path: {}
                        f:port: {}
                        f:scheme: {}
                      f:periodSeconds: {}
                      f:successThreshold: {}
                      f:timeoutSeconds: {}
                    f:resources: {}
                    f:terminationMessagePath: {}
                    f:terminationMessagePolicy: {}
                    f:volumeMounts:
                      .: {}
                      k:{&quot;mountPath&quot;:&quot;/etc/grafana/grafana.ini&quot;}:
                        .: {}
                        f:mountPath: {}
                        f:name: {}
                        f:subPath: {}
                      k:{&quot;mountPath&quot;:&quot;/etc/grafana/provisioning/datasources&quot;}:
                        .: {}
                        f:mountPath: {}
                        f:name: {}
                      k:{&quot;mountPath&quot;:&quot;/var/lib/grafana&quot;}:
                        .: {}
                        f:mountPath: {}
                        f:name: {}
                  k:{&quot;name&quot;:&quot;grafana-sc-datasources&quot;}:
                    .: {}
                    f:env:
                      .: {}
                      k:{&quot;name&quot;:&quot;FOLDER&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;LABEL&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;METHOD&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;REQ_METHOD&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;REQ_PASSWORD&quot;}:
                        .: {}
                        f:name: {}
                        f:valueFrom:
                          .: {}
                          f:secretKeyRef:
                            .: {}
                            f:key: {}
                            f:name: {}
                      k:{&quot;name&quot;:&quot;REQ_URL&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                      k:{&quot;name&quot;:&quot;REQ_USERNAME&quot;}:
                        .: {}
                        f:name: {}
                        f:valueFrom:
                          .: {}
                          f:secretKeyRef:
                            .: {}
                            f:key: {}
                            f:name: {}
                      k:{&quot;name&quot;:&quot;RESOURCE&quot;}:
                        .: {}
                        f:name: {}
                        f:value: {}
                    f:image: {}
                    f:imagePullPolicy: {}
                    f:name: {}
                    f:resources: {}
                    f:terminationMessagePath: {}
                    f:terminationMessagePolicy: {}
                    f:volumeMounts:
                      .: {}
                      k:{&quot;mountPath&quot;:&quot;/etc/grafana/provisioning/datasources&quot;}:
                        .: {}
                        f:mountPath: {}
                        f:name: {}
                f:dnsPolicy: {}
                f:enableServiceLinks: {}
                f:restartPolicy: {}
                f:schedulerName: {}
                f:securityContext:
                  .: {}
                  f:fsGroup: {}
                  f:runAsGroup: {}
                  f:runAsUser: {}
                f:serviceAccount: {}
                f:serviceAccountName: {}
                f:terminationGracePeriodSeconds: {}
                f:volumes:
                  .: {}
                  k:{&quot;name&quot;:&quot;config&quot;}:
                    .: {}
                    f:configMap:
                      .: {}
                      f:defaultMode: {}
                      f:name: {}
                    f:name: {}
                  k:{&quot;name&quot;:&quot;sc-datasources-volume&quot;}:
                    .: {}
                    f:emptyDir: {}
                    f:name: {}
                  k:{&quot;name&quot;:&quot;storage&quot;}:
                    .: {}
                    f:emptyDir: {}
                    f:name: {}
      - manager: node-fetch
        operation: Update
        apiVersion: apps/v1
        time: '2022-06-20T09:09:18Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:spec:
            f:template:
              f:metadata:
                f:annotations:
                  f:kubectl.kubernetes.io/restartedAt: {}
      - manager: kube-controller-manager
        operation: Update
        apiVersion: apps/v1
        time: '2022-06-20T10:02:09Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:annotations:
              f:deployment.kubernetes.io/revision: {}
          f:status:
            f:availableReplicas: {}
            f:conditions:
              .: {}
              k:{&quot;type&quot;:&quot;Available&quot;}:
                .: {}
                f:lastTransitionTime: {}
                f:lastUpdateTime: {}
                f:message: {}
                f:reason: {}
                f:status: {}
                f:type: {}
              k:{&quot;type&quot;:&quot;Progressing&quot;}:
                .: {}
                f:lastTransitionTime: {}
                f:lastUpdateTime: {}
                f:message: {}
                f:reason: {}
                f:status: {}
                f:type: {}
            f:observedGeneration: {}
            f:readyReplicas: {}
            f:replicas: {}
            f:updatedReplicas: {}
    selfLink: /apis/apps/v1/namespaces/grafana-loki/deployments/loki-stack-grafana
  status:
    observedGeneration: 4
    replicas: 1
    updatedReplicas: 1
    readyReplicas: 1
    availableReplicas: 1
    conditions:
      - type: Available
        status: 'True'
        lastUpdateTime: '2022-06-08T06:46:27Z'
        lastTransitionTime: '2022-06-08T06:46:27Z'
        reason: MinimumReplicasAvailable
        message: Deployment has minimum availability.
      - type: Progressing
        status: 'True'
        lastUpdateTime: '2022-06-20T10:02:09Z'
        lastTransitionTime: '2022-06-08T06:45:57Z'
        reason: NewReplicaSetAvailable
        message: ReplicaSet &quot;loki-stack-grafana-7c5d9b4bd5&quot; has successfully progressed.
  spec:
    replicas: 1
    selector:
      matchLabels:
        app.kubernetes.io/instance: loki-stack
        app.kubernetes.io/name: grafana
    template:
      metadata:
        creationTimestamp: null
        labels:
          app.kubernetes.io/instance: loki-stack
          app.kubernetes.io/name: grafana
        annotations:
          checksum/config: c9555bdb452894c521c85985f3b8a62b655d831921884d4a87c2159575c246d3
          checksum/dashboards-json-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
          checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
          checksum/secret: cf6cd98e17ab9ab31c08e2d08258834e3a939d64366f351d1089fa4b9599c3ba
          kubectl.kubernetes.io/restartedAt: '2022-06-20T10:01:43Z'
      spec:
        volumes:
          - name: config
            configMap:
              name: loki-stack-grafana
              defaultMode: 420
          - name: storage
            emptyDir: {}
          - name: sc-datasources-volume
            emptyDir: {}
        containers:
          - name: grafana-sc-datasources
            image: quay.io/kiwigrid/k8s-sidecar:1.15.6
            env:
              - name: METHOD
                value: WATCH
              - name: LABEL
                value: grafana_datasource
              - name: FOLDER
                value: /etc/grafana/provisioning/datasources
              - name: RESOURCE
                value: both
              - name: REQ_USERNAME
                valueFrom:
                  secretKeyRef:
                    name: loki-stack-grafana
                    key: admin-user
              - name: REQ_PASSWORD
                valueFrom:
                  secretKeyRef:
                    name: loki-stack-grafana
                    key: admin-password
              - name: REQ_URL
                value: http://localhost:3000/api/admin/provisioning/datasources/reload
              - name: REQ_METHOD
                value: POST
            resources: {}
            volumeMounts:
              - name: sc-datasources-volume
                mountPath: /etc/grafana/provisioning/datasources
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
            imagePullPolicy: IfNotPresent
          - name: grafana
            image: grafana/grafana:8.3.5
            ports:
              - name: service
                containerPort: 80
                protocol: TCP
              - name: grafana
                containerPort: 3000
                protocol: TCP
            env:
              - name: GF_SECURITY_ADMIN_USER
                valueFrom:
                  secretKeyRef:
                    name: loki-stack-grafana
                    key: admin-user
              - name: GF_SECURITY_ADMIN_PASSWORD
                valueFrom:
                  secretKeyRef:
                    name: loki-stack-grafana
                    key: admin-password
              - name: GF_PATHS_DATA
                value: /var/lib/grafana/
              - name: GF_PATHS_LOGS
                value: /var/log/grafana
              - name: GF_PATHS_PLUGINS
                value: /var/lib/grafana/plugins
              - name: GF_PATHS_PROVISIONING
                value: /etc/grafana/provisioning
            resources: {}
            volumeMounts:
              - name: config
                mountPath: /etc/grafana/grafana.ini
                subPath: grafana.ini
              - name: storage
                mountPath: /var/lib/grafana
              - name: sc-datasources-volume
                mountPath: /etc/grafana/provisioning/datasources
            livenessProbe:
              httpGet:
                path: /api/health
                port: 3000
                scheme: HTTP
              initialDelaySeconds: 60
              timeoutSeconds: 30
              periodSeconds: 10
              successThreshold: 1
              failureThreshold: 10
            readinessProbe:
              httpGet:
                path: /api/health
                port: 3000
                scheme: HTTP
              timeoutSeconds: 1
              periodSeconds: 10
              successThreshold: 1
              failureThreshold: 3
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
            imagePullPolicy: IfNotPresent
        restartPolicy: Always
        terminationGracePeriodSeconds: 30
        dnsPolicy: ClusterFirst
        serviceAccountName: loki-stack-grafana
        serviceAccount: loki-stack-grafana
        automountServiceAccountToken: true
        securityContext:
          runAsUser: 472
          runAsGroup: 472
          fsGroup: 472
        schedulerName: default-scheduler
        enableServiceLinks: true
    strategy:
      type: RollingUpdate
      rollingUpdate:
        maxUnavailable: 25%
        maxSurge: 25%
    revisionHistoryLimit: 10
    progressDeadlineSeconds: 600
  </code></pre>
  <p>This the PVC</p>
  <pre><code>apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    name: storage-loki-stack-0
    namespace: grafana-loki
    uid: efbcac70-785e-4ee5-bf40-2a869349606f
    resourceVersion: '142888329'
    creationTimestamp: '2022-06-08T06:30:21Z'
    labels:
      app: loki
      release: loki-stack
    annotations:
      pv.kubernetes.io/bind-completed: 'yes'
      pv.kubernetes.io/bound-by-controller: 'yes'
      volume.beta.kubernetes.io/storage-provisioner: dobs.csi.digitalocean.com
    finalizers:
      - kubernetes.io/pvc-protection
    managedFields:
      - manager: kube-controller-manager
        operation: Update
        apiVersion: v1
        time: '2022-06-08T06:30:25Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:annotations:
              .: {}
              f:pv.kubernetes.io/bind-completed: {}
              f:pv.kubernetes.io/bound-by-controller: {}
              f:volume.beta.kubernetes.io/storage-provisioner: {}
            f:labels:
              .: {}
              f:app: {}
              f:release: {}
          f:spec:
            f:accessModes: {}
            f:resources:
              f:requests:
                .: {}
                f:storage: {}
            f:volumeMode: {}
            f:volumeName: {}
          f:status:
            f:accessModes: {}
            f:capacity:
              .: {}
              f:storage: {}
            f:phase: {}
    selfLink: /api/v1/namespaces/grafana-loki/persistentvolumeclaims/storage-loki-stack-0
  status:
    phase: Bound
    accessModes:
      - ReadWriteOnce
    capacity:
      storage: 10Gi
  spec:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: 10Gi
    volumeName: pvc-efbcac70-785e-4ee5-bf40-2a869349606f
    storageClassName: do-block-storage
    volumeMode: Filesystem
  </code></pre>
  <p>This is PV</p>
  <pre><code>apiVersion: v1
  kind: PersistentVolume
  metadata:
    name: pvc-efbcac70-785e-4ee5-bf40-2a869349606f
    uid: c91f45dc-b0c2-4077-a76c-16c83256587c
    resourceVersion: '142888376'
    creationTimestamp: '2022-06-08T06:30:25Z'
    annotations:
      pv.kubernetes.io/provisioned-by: dobs.csi.digitalocean.com
    finalizers:
      - kubernetes.io/pv-protection
      - external-attacher/dobs-csi-digitalocean-com
    managedFields:
      - manager: csi-provisioner
        operation: Update
        apiVersion: v1
        time: '2022-06-08T06:30:25Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:annotations:
              .: {}
              f:pv.kubernetes.io/provisioned-by: {}
          f:spec:
            f:accessModes: {}
            f:capacity:
              .: {}
              f:storage: {}
            f:claimRef:
              .: {}
              f:apiVersion: {}
              f:kind: {}
              f:name: {}
              f:namespace: {}
              f:resourceVersion: {}
              f:uid: {}
            f:csi:
              .: {}
              f:driver: {}
              f:fsType: {}
              f:volumeAttributes:
                .: {}
                f:storage.kubernetes.io/csiProvisionerIdentity: {}
              f:volumeHandle: {}
            f:persistentVolumeReclaimPolicy: {}
            f:storageClassName: {}
            f:volumeMode: {}
      - manager: kube-controller-manager
        operation: Update
        apiVersion: v1
        time: '2022-06-08T06:30:25Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:status:
            f:phase: {}
      - manager: csi-attacher
        operation: Update
        apiVersion: v1
        time: '2022-06-08T06:30:41Z'
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:finalizers:
              v:&quot;external-attacher/dobs-csi-digitalocean-com&quot;: {}
    selfLink: /api/v1/persistentvolumes/pvc-efbcac70-785e-4ee5-bf40-2a869349606f
  status:
    phase: Bound
  spec:
    capacity:
      storage: 10Gi
    csi:
      driver: dobs.csi.digitalocean.com
      volumeHandle: 7b1253ab-e6f4-11ec-9c18-0a58ac14b6a4
      fsType: ext4
      volumeAttributes:
        storage.kubernetes.io/csiProvisionerIdentity: 1651923394559-8081-dobs.csi.digitalocean.com
    accessModes:
      - ReadWriteOnce
    claimRef:
      kind: PersistentVolumeClaim
      namespace: grafana-loki
      name: storage-loki-stack-0
      uid: efbcac70-785e-4ee5-bf40-2a869349606f
      apiVersion: v1
      resourceVersion: '142888304'
    persistentVolumeReclaimPolicy: Delete
    storageClassName: do-block-storage
    volumeMode: Filesystem
  </code></pre>
  <p>Let me know if anything else is needed. thank you</p>