Elasticsearch Backup

$ kubectl get secrets/gv-es-elastic-user -o go-template='{{.data.elastic | base64decode}}{{"\n"}}'
3jdE4sAs81aL82907fkTtVa7

$ kubectl get secret gv-essentials-minio -o go-template='{{.data.rootUser | base64decode}}{{"\n"}}'
NsdlLiCEZBHRLzIb5PdI
$

$ kubectl get secret gv-essentials-minio -o go-template='{{.data.rootPassword | base64decode}}{{"\n"}}'
fFeIh8VyxRf45HUIJDgQxn0AcViJV20qa5cRkiP1
$

$ kubectl run -it --rm --image=docker.io/getvisibility/backup:0.1.4 debug -- /bin/bash
If you don't see a command prompt, try pressing enter.
root@debug:/# mc alias set myminio http://gv-essentials-minio:9000 NsdlLiCEZBHRLzIb5PdI fFeIh8VyxRf45HUIJDgQxn0AcViJV20qa5cRkiP1
mc: Configuration written to `/root/.mc/config.json`. Please update your access credentials.
mc: Successfully created `/root/.mc/share`.
mc: Initialized share uploads `/root/.mc/share/uploads.json` file.
mc: Initialized share downloads `/root/.mc/share/downloads.json` file.
Added `myminio` successfully.
root@debug:/# curl -X POST -u "elastic:3jdE4sAs81aL82907fkTtVa7" "http://gv-es-http:9200/_slm/policy/gv-essentials-minio/_execute"
{"snapshot_name":"gv-essentials-minio-j6rghefwtuexdlg07uqauq"}
root@debug:/#

$ kubectl run -it --rm --image=docker.io/getvisibility/backup:0.1.4 debug -- /bin/bash
If you don't see a command prompt, try pressing enter.
root@debug:/# mc alias set myminio http://gv-essentials-minio:9000 NsdlLiCEZBHRLzIb5PdI fFeIh8VyxRf45HUIJDgQxn0AcViJV20qa5cRkiP1
mc: Configuration written to `/root/.mc/config.json`. Please update your access credentials.
mc: Successfully created `/root/.mc/share`.
mc: Initialized share uploads `/root/.mc/share/uploads.json` file.
mc: Initialized share downloads `/root/.mc/share/downloads.json` file.
Added `myminio` successfully.
root@debug:/# curl -X POST -u "elastic:FmHaDrJeGkLDfHoSrFeFvJeQrN" "http://gv-es-http:9200/_slm/policy/gv-essentials-minio/_execute?pretty"
{
  "gv-essentials-minio" : {
    "version" : 5,
    "modified_date_millis" : 1729816595792,
    "policy" : {
      "name" : "gv-essentials-minio",
      "schedule" : "0 0 2 * * ?",
      "repository" : "gv-essentials-minio",
      "config" : {
        "indices" : "*",
        "include_global_state" : true
      },
      "retention" : {
        "expire_after" : "30d",
        "min_count" : 5,
        "max_count" : 40
      }
    },
    "last_success" : {
      "snapshot_name" : "gv-essentials-minio-w518btqdsuahriw7pqw9qq",
      "start_time" : 1730132687834,
      "time" : 1730132697502
    },
    "next_execution_millis" : 1730167200000,
    "stats" : {
      "policy" : "gv-essentials-minio",
      "snapshots_taken" : 8,
      "snapshots_failed" : 0,
      "snapshots_deleted" : 0,
      "snapshot_deletion_failures" : 0
    }
  }
}

ubuntu@marek-lab:~/gv-backup$ kubectl get secrets/gv-es-elastic-user -o go-template='{{.data.elastic | base64decode}}{{"\n"}}'
73tI55x719TF1T4eb6TLdqeQ
ubuntu@marek-lab:~/gv-backup$ kubectl exec -it minio-backup-pod -- /bin/bash
root@minio-backup-pod:/# mc alias set myminio http://gv-essentials-minio.default.svc.cluster.local:9000 4fBmdCHECPdRU9LsxhcI SiblO9qh3U1abhJAb8UEIEUbl9xRLWf29Drk26xY
Added `myminio` successfully.
root@minio-backup-pod:/# export ES_PASSWORD=73tI55x719TF1T4eb6TLdqeQ
root@minio-backup-pod:/# 

cat > gv-es-backup.sh << 'EOL'
#!/bin/bash

# Configuration
ES_HOST="http://gv-es-http:9200"
ES_USER="elastic"
BACKUP_DIR="/tmp/es-backup"
REPO_NAME="gv-essentials-minio"
MINIO_BUCKET="elasticsearch-backup"
MAX_WAIT_SECONDS=600  # 10 minutes timeout

# Function to check MinIO connectivity
check_minio_connection() {
    echo "Testing MinIO connection..."
    MINIO_OUTPUT=$(mc ls myminio 2>&1)
    if [ -z "$MINIO_OUTPUT" ]; then
        printf "\n\n\n\n\n\n\nError:\n\n  MinIO is not configured or not accessible.\n  Please ensure:\n  1. MinIO client (mc) is installed\n  2. 'myminio' alias is configured\n  3. MinIO server is running\n\nTo configure MinIO, run:\n  mc alias set myminio <MINIO_URL> <ACCESS_KEY> <SECRET_KEY>\n\n"
        exit 1
    fi
    
    echo "Testing MinIO bucket access..."
    BUCKET_OUTPUT=$(mc ls "myminio/${MINIO_BUCKET}" 2>&1)
    if [ -z "$BUCKET_OUTPUT" ]; then
        printf '\n\n\n\n\n\n\nError:\n\n  Cannot access bucket %s.\nPlease ensure:\n  1. Bucket exists\n  2. You have proper permissions\n\n' "$MINIO_BUCKET"
        exit 1
    fi
    echo "MinIO connection successful"
}

check_elasticsearch_connection() {
    echo "Testing Elasticsearch connection..."
    if ! curl -s -u "${ES_USER}:${ES_PASSWORD}" "${ES_HOST}/_cluster/health" > /dev/null; then
        printf "\n\n\n\n\n\n\nError:\n\n  Failed to connect to Elasticsearch. Invalid credentials or connection issue.\n\n"
        unset ES_PASSWORD
        exit 1
    fi
}

# Password handling
check_password() {
    printf "gv-es-backup.sh\n------------------------\n\n"
    if [ -z "${ES_PASSWORD}" ]; then
        printf '\n\n\n\n\n\n\nError:\n\n  ES_PASSWORD environment variable is not set.\n  In terminal, run:\n  export ES_PASSWORD=<password>\n\n  And then run this script again.\n\n'
        exit 1
    fi

}

# Password handling
check_password() {
    printf "gv-es-backup.sh\n------------------------\n\n"
    if [ -z "${ES_PASSWORD}" ]; then
        printf "\n\n\n\n\n\n\nError:\nES_PASSWORD environment variable is not set.\nIn terminal, run:\n  export ES_PASSWORD=<password>\n\nAnd then run this script again.\n\n"
        exit 1
    fi

}

# Function to find the current index file
find_index_file() {
    index=$(mc ls myminio/elasticsearch-backup/ | \
    awk '/index-/ {print $6}' | \
    sort -V | \
    tail -n 1)
    echo "Index: [$index]" > index.latest
    echo $index
}

# Function to check if snapshot is complete
check_snapshot_status() {
    local snapshot_name=$1
    local status=$(curl -s -u "${ES_USER}:${ES_PASSWORD}" "${ES_HOST}/_snapshot/${REPO_NAME}/${snapshot_name}/_status" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)
    echo $status
}

# Initial checks
check_password
check_minio_connection
check_elasticsearch_connection  

echo "Starting backup process..."

# 1. Execute snapshot policy
echo "Triggering snapshot creation..."
SNAPSHOT_RESPONSE=$(curl -s -X POST -u "${ES_USER}:${ES_PASSWORD}" "${ES_HOST}/_slm/policy/${REPO_NAME}/_execute?pretty")
SNAPSHOT_NAME=$(echo "$SNAPSHOT_RESPONSE" | grep '"snapshot_name"' | sed -E 's/.*"snapshot_name"\s*:\s*"([^"]+)".*/\1/')

if [ -z "$SNAPSHOT_NAME" ]; then
    echo "  Failed to get snapshot name. Exiting."
    exit 1
fi

echo "Initiated snapshot: ${SNAPSHOT_NAME}"

# 2. Wait for snapshot completion
echo "  Waiting for snapshot to complete..."
SECONDS_WAITED=0
while [ $SECONDS_WAITED -lt $MAX_WAIT_SECONDS ]; do
    STATUS=$(check_snapshot_status "$SNAPSHOT_NAME")
    
    if [ "$STATUS" = "SUCCESS" ]; then
        echo "  Snapshot completed successfully"
        break
    elif [ "$STATUS" = "FAILED" ]; then
        echo "  Snapshot failed"
        exit 1
    fi
    
    sleep 10
    SECONDS_WAITED=$((SECONDS_WAITED + 10))
    echo "  Still waiting... ($SECONDS_WAITED seconds)"
done

# Find current index file
CURRENT_INDEX=$(find_index_file)
if [ -z "${CURRENT_INDEX}" ]; then
    echo "  Failed to find current index file. Exiting."
    exit 1
fi
echo "Current index file: ${CURRENT_INDEX}"

if [ $SECONDS_WAITED -ge $MAX_WAIT_SECONDS ]; then
    echo "Timeout waiting for snapshot completion"
    exit 1
fi

# 3. Get UUID for the snapshot
echo "Retrieving snapshot UUID..."

UUID=$(curl -s -u "${ES_USER}:${ES_PASSWORD}" "${ES_HOST}/_snapshot/${REPO_NAME}/${SNAPSHOT_NAME}?pretty" | grep "uuid" | awk -F'"' '{print $4}')

if [ -z "$UUID" ]; then
    echo "  Failed to get UUID. Exiting."
    exit 1
fi

echo "  Found UUID: ${UUID}"

# 4. Create backup directory and copy files
rm -rf "${BACKUP_DIR}"
mkdir -p "${BACKUP_DIR}"

echo "Copying snapshot files..."
echo "Required files:"
echo "- meta-${UUID}.dat"
echo "- snap-${UUID}.dat"
echo "- index.latest"
echo "- ${CURRENT_INDEX}"

# Copy files with verification
for file in "meta-${UUID}.dat" "snap-${UUID}.dat" "index.latest" "${CURRENT_INDEX}"; do
    echo "Copying $file..."
    if ! mc ls "myminio/${MINIO_BUCKET}/${file}" >/dev/null 2>&1; then
        echo "Warning: File $file not found in MinIO"
        continue
    fi
    mc cp "myminio/${MINIO_BUCKET}/${file}" "${BACKUP_DIR}/" || { echo "Failed to copy $file"; exit 1; }
done

cd /tmp/es-backup
printf "\n\nList of backed up Elasticsearch content:\n"
find . -type f -exec du -k {} \; | awk '{printf "%d KB\t%s\n", $1, $2}' | sort -n
printf "\n"
cd /

# 5. Create archive
ARCHIVE_NAME="es-snapshot.tar.gz"

echo "Creating archive ${ARCHIVE_NAME}..."
tar -czf "${ARCHIVE_NAME}" -C "${BACKUP_DIR}" .

# Cleanup
# rm -rf "${BACKUP_DIR}"

echo "Backup process completed successfully"
echo "Archive created: ${ARCHIVE_NAME}"
ls -lh "${ARCHIVE_NAME}"
echo "----------------------------------------"
echo "Done."
EOL
chmod +x gv-es-backup.sh && clear && ./gv-es-backup.sh

Testing Elasticsearch connection...
Starting backup process...
Triggering snapshot creation...
Initiated snapshot: gv-essentials-minio-qc36iwyfrckx9k2tupukiw
Waiting for snapshot to complete...
Still waiting... (10 seconds)
Snapshot completed successfully
Current index file: index-35
Retrieving snapshot UUID...
Found UUID: xqY3LPdNRr-vubBCUaZGAA
Copying snapshot files...
Required files:
- meta-xqY3LPdNRr-vubBCUaZGAA.dat
- snap-xqY3LPdNRr-vubBCUaZGAA.dat
- index.latest
- index-35
Copying meta-xqY3LPdNRr-vubBCUaZGAA.dat...
...vubBCUaZGAA.dat: 28.57 KiB / 28.57 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 2.94 MiB/s 0s
Copying snap-xqY3LPdNRr-vubBCUaZGAA.dat...
...vubBCUaZGAA.dat: 1.86 KiB / 1.86 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 209.10 KiB/s 0s
Copying index.latest...
...up/index.latest: 8 B / 8 B ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 868 B/s 0s
Copying index-35...
...backup/index-35: 230.52 KiB / 230.52 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 22.74 MiB/s 0s
Creating archive es-snapshot.tar.gz...
Backup process completed successfully
Archive created: es-snapshot.tar.gz
-rw-r--r-- 1 root root 45K Nov 10 03:35 es-snapshot.tar.gz
root@minio-backup-pod:/# 

root@minio-backup-pod:/# exit
ubuntu@marek-lab:~/gv-backup$ kubectl cp minio-backup-pod:es-snapshot.tar.gz full-backup/es-snapshot.tar.gz 
ubuntu@marek-lab:~/gv-backup$ ls -lh full-backup/
total 164K
-rw-rw-r-- 1 ubuntu ubuntu 3.8K Nov 10 00:35 consul-snapshot.snap
-rw-rw-r-- 1 ubuntu ubuntu  47K Nov 10 05:19 es-snapshot.tar.gz
-rw-rw-r-- 1 ubuntu ubuntu 109K Nov  9 21:54 postgres-backup.sql.gz
ubuntu@marek-lab:~/gv-backup$ 

1. Navigate to https://10.20.30.40/gv-essentials-minio/ and enter username/password.

   

   Important:

   First we need to retrieve Elasticsearch password. You can retrieve Elasticsearch password from either Rancher or using the terminal. In Rancher, find your cluster and navigate to Storage > Secrets > gv-es-elastic-user.

   Note: Username is gv.

   

   In terminal, use:

   $ kubectl get secrets/gv-es-elastic-user -o go-template='{{.data.elastic | base64decode}}{{"\n"}}'
   3jdE4sAs81aL82907fkTtVa7

   After logging in, you will see the main screen of the MinIO Console.

   

   PostgreSQL stores multiple databases, each backed up individually, and a complete backup requires including all of them.

   

2. Click Download to download the backups directly from the browser using the MinIO console.